public virtual void TestMRTimelineEventHandling()
{
Configuration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
conf.SetBoolean(MRJobConfig.MapreduceJobEmitTimelineData, true);
MiniMRYarnCluster cluster = null;
try
{
cluster = new MiniMRYarnCluster(typeof(TestJobHistoryEventHandler).Name, 1);
cluster.Init(conf);
cluster.Start();
conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname
() + ":" + cluster.GetApplicationHistoryServer().GetPort());
TimelineStore ts = cluster.GetApplicationHistoryServer().GetTimelineStore();
Path inDir = new Path("input");
Path outDir = new Path("output");
RunningJob job = UtilsForTests.RunJobSucceed(new JobConf(conf), inDir, outDir);
NUnit.Framework.Assert.AreEqual(JobStatus.Succeeded, job.GetJobStatus().GetState(
).GetValue());
TimelineEntities entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null
, null, null, null, null, null);
NUnit.Framework.Assert.AreEqual(1, entities.GetEntities().Count);
TimelineEntity tEntity = entities.GetEntities()[0];
NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId());
NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType());
NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents
()[tEntity.GetEvents().Count - 1].GetEventType());
NUnit.Framework.Assert.AreEqual(EventType.JobFinished.ToString(), tEntity.GetEvents
()[0].GetEventType());
job = UtilsForTests.RunJobFail(new JobConf(conf), inDir, outDir);
NUnit.Framework.Assert.AreEqual(JobStatus.Failed, job.GetJobStatus().GetState().GetValue
());
entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null
, null, null);
NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count);
tEntity = entities.GetEntities()[0];
NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId());
NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType());
NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents
()[tEntity.GetEvents().Count - 1].GetEventType());
NUnit.Framework.Assert.AreEqual(EventType.JobFailed.ToString(), tEntity.GetEvents
()[0].GetEventType());
}
finally
{
if (cluster != null)
{
cluster.Stop();
}
}
}
public virtual void TestPutEntities()
{
TestTimelineWebServicesWithSSL.TestTimelineClient client = new TestTimelineWebServicesWithSSL.TestTimelineClient
();
try
{
client.Init(conf);
client.Start();
TimelineEntity expectedEntity = new TimelineEntity();
expectedEntity.SetEntityType("test entity type");
expectedEntity.SetEntityId("test entity id");
expectedEntity.SetDomainId("test domain id");
TimelineEvent @event = new TimelineEvent();
@event.SetEventType("test event type");
@event.SetTimestamp(0L);
expectedEntity.AddEvent(@event);
TimelinePutResponse response = client.PutEntities(expectedEntity);
NUnit.Framework.Assert.AreEqual(0, response.GetErrors().Count);
NUnit.Framework.Assert.IsTrue(client.resp.ToString().Contains("https"));
TimelineEntity actualEntity = store.GetEntity(expectedEntity.GetEntityId(), expectedEntity
.GetEntityType(), EnumSet.AllOf <TimelineReader.Field>());
NUnit.Framework.Assert.IsNotNull(actualEntity);
NUnit.Framework.Assert.AreEqual(expectedEntity.GetEntityId(), actualEntity.GetEntityId
());
NUnit.Framework.Assert.AreEqual(expectedEntity.GetEntityType(), actualEntity.GetEntityType
());
}
finally
{
client.Stop();
client.Close();
}
}
public virtual void TestGetOldEntityWithOutDomainId()
{
TimelineEntity entity = dataManaer.GetEntity("OLD_ENTITY_TYPE_1", "OLD_ENTITY_ID_1"
, null, UserGroupInformation.GetCurrentUser());
NUnit.Framework.Assert.IsNotNull(entity);
NUnit.Framework.Assert.AreEqual("OLD_ENTITY_ID_1", entity.GetEntityId());
NUnit.Framework.Assert.AreEqual("OLD_ENTITY_TYPE_1", entity.GetEntityType());
NUnit.Framework.Assert.AreEqual(TimelineDataManager.DefaultDomainId, entity.GetDomainId
());
}
private static TimelineEntity MaskFields(TimelineEntity entity, EnumSet <TimelineReader.Field
> fields)
{
// Conceal the fields that are not going to be exposed
TimelineEntity entityToReturn = new TimelineEntity();
entityToReturn.SetEntityId(entity.GetEntityId());
entityToReturn.SetEntityType(entity.GetEntityType());
entityToReturn.SetStartTime(entity.GetStartTime());
entityToReturn.SetDomainId(entity.GetDomainId());
// Deep copy
if (fields.Contains(TimelineReader.Field.Events))
{
entityToReturn.AddEvents(entity.GetEvents());
}
else
{
if (fields.Contains(TimelineReader.Field.LastEventOnly))
{
entityToReturn.AddEvent(entity.GetEvents()[0]);
}
else
{
entityToReturn.SetEvents(null);
}
}
if (fields.Contains(TimelineReader.Field.RelatedEntities))
{
entityToReturn.AddRelatedEntities(entity.GetRelatedEntities());
}
else
{
entityToReturn.SetRelatedEntities(null);
}
if (fields.Contains(TimelineReader.Field.PrimaryFilters))
{
entityToReturn.AddPrimaryFilters(entity.GetPrimaryFilters());
}
else
{
entityToReturn.SetPrimaryFilters(null);
}
if (fields.Contains(TimelineReader.Field.OtherInfo))
{
entityToReturn.AddOtherInfo(entity.GetOtherInfo());
}
else
{
entityToReturn.SetOtherInfo(null);
}
return(entityToReturn);
}
private void PutEntity(TimelineEntity entity)
{
try
{
if (Log.IsDebugEnabled())
{
Log.Debug("Publishing the entity " + entity.GetEntityId() + ", JSON-style content: "
+ TimelineUtils.DumpTimelineRecordtoJSON(entity));
}
client.PutEntities(entity);
}
catch (Exception e)
{
Log.Error("Error when publishing entity [" + entity.GetEntityType() + "," + entity
.GetEntityId() + "]", e);
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual bool CheckAccess(UserGroupInformation callerUGI, ApplicationAccessType
applicationAccessType, TimelineEntity entity)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Verifying the access of " + (callerUGI == null ? null : callerUGI.GetShortUserName
()) + " on the timeline entity " + new EntityIdentifier(entity.GetEntityId(), entity
.GetEntityType()));
}
if (!adminAclsManager.AreACLsEnabled())
{
return(true);
}
// find domain owner and acls
TimelineACLsManager.AccessControlListExt aclExt = aclExts[entity.GetDomainId()];
if (aclExt == null)
{
aclExt = LoadDomainFromTimelineStore(entity.GetDomainId());
}
if (aclExt == null)
{
throw new YarnException("Domain information of the timeline entity " + new EntityIdentifier
(entity.GetEntityId(), entity.GetEntityType()) + " doesn't exist.");
}
string owner = aclExt.owner;
AccessControlList domainACL = aclExt.acls[applicationAccessType];
if (domainACL == null)
{
if (Log.IsDebugEnabled())
{
Log.Debug("ACL not found for access-type " + applicationAccessType + " for domain "
+ entity.GetDomainId() + " owned by " + owner + ". Using default [" + YarnConfiguration
.DefaultYarnAppAcl + "]");
}
domainACL = new AccessControlList(YarnConfiguration.DefaultYarnAppAcl);
}
if (callerUGI != null && (adminAclsManager.IsAdmin(callerUGI) || callerUGI.GetShortUserName
().Equals(owner) || domainACL.IsUserAllowed(callerUGI)))
{
return(true);
}
return(false);
}
/// <exception cref="System.IO.IOException"/>
public virtual TimelineEntities GetEntities(string entityType, long limit, long windowStart
, long windowEnd, string fromId, long fromTs, NameValuePair primaryFilter, ICollection
<NameValuePair> secondaryFilters, EnumSet <TimelineReader.Field> fields, TimelineDataManager.CheckAcl
checkAcl)
{
lock (this)
{
if (limit == null)
{
limit = DefaultLimit;
}
if (windowStart == null)
{
windowStart = long.MinValue;
}
if (windowEnd == null)
{
windowEnd = long.MaxValue;
}
if (fields == null)
{
fields = EnumSet.AllOf <TimelineReader.Field>();
}
IEnumerator <TimelineEntity> entityIterator = null;
if (fromId != null)
{
TimelineEntity firstEntity = entities[new EntityIdentifier(fromId, entityType)];
if (firstEntity == null)
{
return(new TimelineEntities());
}
else
{
entityIterator = new TreeSet <TimelineEntity>(entities.Values).TailSet(firstEntity
, true).GetEnumerator();
}
}
if (entityIterator == null)
{
entityIterator = new PriorityQueue <TimelineEntity>(entities.Values).GetEnumerator
();
}
IList <TimelineEntity> entitiesSelected = new AList <TimelineEntity>();
while (entityIterator.HasNext())
{
TimelineEntity entity = entityIterator.Next();
if (entitiesSelected.Count >= limit)
{
break;
}
if (!entity.GetEntityType().Equals(entityType))
{
continue;
}
if (entity.GetStartTime() <= windowStart)
{
continue;
}
if (entity.GetStartTime() > windowEnd)
{
continue;
}
if (fromTs != null && entityInsertTimes[new EntityIdentifier(entity.GetEntityId()
, entity.GetEntityType())] > fromTs)
{
continue;
}
if (primaryFilter != null && !MatchPrimaryFilter(entity.GetPrimaryFilters(), primaryFilter
))
{
continue;
}
if (secondaryFilters != null)
{
// AND logic
bool flag = true;
foreach (NameValuePair secondaryFilter in secondaryFilters)
{
if (secondaryFilter != null && !MatchPrimaryFilter(entity.GetPrimaryFilters(), secondaryFilter
) && !MatchFilter(entity.GetOtherInfo(), secondaryFilter))
{
flag = false;
break;
}
}
if (!flag)
{
continue;
}
}
if (entity.GetDomainId() == null)
{
entity.SetDomainId(TimelineDataManager.DefaultDomainId);
}
if (checkAcl == null || checkAcl.Check(entity))
{
entitiesSelected.AddItem(entity);
}
}
IList <TimelineEntity> entitiesToReturn = new AList <TimelineEntity>();
foreach (TimelineEntity entitySelected in entitiesSelected)
{
entitiesToReturn.AddItem(MaskFields(entitySelected, fields));
}
entitiesToReturn.Sort();
TimelineEntities entitiesWrapper = new TimelineEntities();
entitiesWrapper.SetEntities(entitiesToReturn);
return(entitiesWrapper);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestPublishApplicationMetrics()
{
for (int i = 1; i <= 2; ++i)
{
ApplicationId appId = ApplicationId.NewInstance(0, i);
RMApp app = CreateRMApp(appId);
metricsPublisher.AppCreated(app, app.GetStartTime());
metricsPublisher.AppFinished(app, RMAppState.Finished, app.GetFinishTime());
if (i == 1)
{
metricsPublisher.AppACLsUpdated(app, "uers1,user2", 4L);
}
else
{
// in case user doesn't specify the ACLs
metricsPublisher.AppACLsUpdated(app, null, 4L);
}
TimelineEntity entity = null;
do
{
entity = store.GetEntity(appId.ToString(), ApplicationMetricsConstants.EntityType
, EnumSet.AllOf <TimelineReader.Field>());
}while (entity == null || entity.GetEvents().Count < 3);
// ensure three events are both published before leaving the loop
// verify all the fields
NUnit.Framework.Assert.AreEqual(ApplicationMetricsConstants.EntityType, entity.GetEntityType
());
NUnit.Framework.Assert.AreEqual(app.GetApplicationId().ToString(), entity.GetEntityId
());
NUnit.Framework.Assert.AreEqual(app.GetName(), entity.GetOtherInfo()[ApplicationMetricsConstants
.NameEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetQueue(), entity.GetOtherInfo()[ApplicationMetricsConstants
.QueueEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetUser(), entity.GetOtherInfo()[ApplicationMetricsConstants
.UserEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetApplicationType(), entity.GetOtherInfo()[ApplicationMetricsConstants
.TypeEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetSubmitTime(), entity.GetOtherInfo()[ApplicationMetricsConstants
.SubmittedTimeEntityInfo]);
if (i == 1)
{
NUnit.Framework.Assert.AreEqual("uers1,user2", entity.GetOtherInfo()[ApplicationMetricsConstants
.AppViewAclsEntityInfo]);
}
else
{
NUnit.Framework.Assert.AreEqual(string.Empty, entity.GetOtherInfo()[ApplicationMetricsConstants
.AppViewAclsEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetRMAppMetrics().GetMemorySeconds(), long.Parse
(entity.GetOtherInfo()[ApplicationMetricsConstants.AppMemMetrics].ToString()));
NUnit.Framework.Assert.AreEqual(app.GetRMAppMetrics().GetVcoreSeconds(), long.Parse
(entity.GetOtherInfo()[ApplicationMetricsConstants.AppCpuMetrics].ToString()));
}
bool hasCreatedEvent = false;
bool hasFinishedEvent = false;
bool hasACLsUpdatedEvent = false;
foreach (TimelineEvent @event in entity.GetEvents())
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.CreatedEventType))
{
hasCreatedEvent = true;
NUnit.Framework.Assert.AreEqual(app.GetStartTime(), @event.GetTimestamp());
}
else
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.FinishedEventType))
{
hasFinishedEvent = true;
NUnit.Framework.Assert.AreEqual(app.GetFinishTime(), @event.GetTimestamp());
NUnit.Framework.Assert.AreEqual(app.GetDiagnostics().ToString(), @event.GetEventInfo
()[ApplicationMetricsConstants.DiagnosticsInfoEventInfo]);
NUnit.Framework.Assert.AreEqual(app.GetFinalApplicationStatus().ToString(), @event
.GetEventInfo()[ApplicationMetricsConstants.FinalStatusEventInfo]);
NUnit.Framework.Assert.AreEqual(YarnApplicationState.Finished.ToString(), @event.
GetEventInfo()[ApplicationMetricsConstants.StateEventInfo]);
}
else
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.AclsUpdatedEventType
))
{
hasACLsUpdatedEvent = true;
NUnit.Framework.Assert.AreEqual(4L, @event.GetTimestamp());
}
}
}
}
NUnit.Framework.Assert.IsTrue(hasCreatedEvent && hasFinishedEvent && hasACLsUpdatedEvent
);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestPublishContainerMetrics()
{
ContainerId containerId = ContainerId.NewContainerId(ApplicationAttemptId.NewInstance
(ApplicationId.NewInstance(0, 1), 1), 1);
RMContainer container = CreateRMContainer(containerId);
metricsPublisher.ContainerCreated(container, container.GetCreationTime());
metricsPublisher.ContainerFinished(container, container.GetFinishTime());
TimelineEntity entity = null;
do
{
entity = store.GetEntity(containerId.ToString(), ContainerMetricsConstants.EntityType
, EnumSet.AllOf <TimelineReader.Field>());
}while (entity == null || entity.GetEvents().Count < 2);
// ensure two events are both published before leaving the loop
// verify all the fields
NUnit.Framework.Assert.AreEqual(ContainerMetricsConstants.EntityType, entity.GetEntityType
());
NUnit.Framework.Assert.AreEqual(containerId.ToString(), entity.GetEntityId());
NUnit.Framework.Assert.AreEqual(containerId.GetApplicationAttemptId().ToString(),
entity.GetPrimaryFilters()[ContainerMetricsConstants.ParentPrimariyFilter].GetEnumerator
().Next());
NUnit.Framework.Assert.AreEqual(container.GetAllocatedNode().GetHost(), entity.GetOtherInfo
()[ContainerMetricsConstants.AllocatedHostEntityInfo]);
NUnit.Framework.Assert.AreEqual(container.GetAllocatedNode().GetPort(), entity.GetOtherInfo
()[ContainerMetricsConstants.AllocatedPortEntityInfo]);
NUnit.Framework.Assert.AreEqual(container.GetAllocatedResource().GetMemory(), entity
.GetOtherInfo()[ContainerMetricsConstants.AllocatedMemoryEntityInfo]);
NUnit.Framework.Assert.AreEqual(container.GetAllocatedResource().GetVirtualCores(
), entity.GetOtherInfo()[ContainerMetricsConstants.AllocatedVcoreEntityInfo]);
NUnit.Framework.Assert.AreEqual(container.GetAllocatedPriority().GetPriority(), entity
.GetOtherInfo()[ContainerMetricsConstants.AllocatedPriorityEntityInfo]);
bool hasCreatedEvent = false;
bool hasFinishedEvent = false;
foreach (TimelineEvent @event in entity.GetEvents())
{
if (@event.GetEventType().Equals(ContainerMetricsConstants.CreatedEventType))
{
hasCreatedEvent = true;
NUnit.Framework.Assert.AreEqual(container.GetCreationTime(), @event.GetTimestamp(
));
}
else
{
if (@event.GetEventType().Equals(ContainerMetricsConstants.FinishedEventType))
{
hasFinishedEvent = true;
NUnit.Framework.Assert.AreEqual(container.GetFinishTime(), @event.GetTimestamp());
NUnit.Framework.Assert.AreEqual(container.GetDiagnosticsInfo(), @event.GetEventInfo
()[ContainerMetricsConstants.DiagnosticsInfoEventInfo]);
NUnit.Framework.Assert.AreEqual(container.GetContainerExitStatus(), @event.GetEventInfo
()[ContainerMetricsConstants.ExitStatusEventInfo]);
NUnit.Framework.Assert.AreEqual(container.GetContainerState().ToString(), @event.
GetEventInfo()[ContainerMetricsConstants.StateEventInfo]);
}
}
}
NUnit.Framework.Assert.IsTrue(hasCreatedEvent && hasFinishedEvent);
}
/// <exception cref="System.Exception"/>
public virtual void TestPublishAppAttemptMetrics()
{
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(ApplicationId
.NewInstance(0, 1), 1);
RMAppAttempt appAttempt = CreateRMAppAttempt(appAttemptId);
metricsPublisher.AppAttemptRegistered(appAttempt, int.MaxValue + 1L);
RMApp app = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
.Undefined);
metricsPublisher.AppAttemptFinished(appAttempt, RMAppAttemptState.Finished, app,
int.MaxValue + 2L);
TimelineEntity entity = null;
do
{
entity = store.GetEntity(appAttemptId.ToString(), AppAttemptMetricsConstants.EntityType
, EnumSet.AllOf <TimelineReader.Field>());
}while (entity == null || entity.GetEvents().Count < 2);
// ensure two events are both published before leaving the loop
// verify all the fields
NUnit.Framework.Assert.AreEqual(AppAttemptMetricsConstants.EntityType, entity.GetEntityType
());
NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), entity.GetEntityId());
NUnit.Framework.Assert.AreEqual(appAttemptId.GetApplicationId().ToString(), entity
.GetPrimaryFilters()[AppAttemptMetricsConstants.ParentPrimaryFilter].GetEnumerator
().Next());
bool hasRegisteredEvent = false;
bool hasFinishedEvent = false;
foreach (TimelineEvent @event in entity.GetEvents())
{
if (@event.GetEventType().Equals(AppAttemptMetricsConstants.RegisteredEventType))
{
hasRegisteredEvent = true;
NUnit.Framework.Assert.AreEqual(appAttempt.GetHost(), @event.GetEventInfo()[AppAttemptMetricsConstants
.HostEventInfo]);
NUnit.Framework.Assert.AreEqual(appAttempt.GetRpcPort(), @event.GetEventInfo()[AppAttemptMetricsConstants
.RpcPortEventInfo]);
NUnit.Framework.Assert.AreEqual(appAttempt.GetMasterContainer().GetId().ToString(
), @event.GetEventInfo()[AppAttemptMetricsConstants.MasterContainerEventInfo]);
}
else
{
if (@event.GetEventType().Equals(AppAttemptMetricsConstants.FinishedEventType))
{
hasFinishedEvent = true;
NUnit.Framework.Assert.AreEqual(appAttempt.GetDiagnostics(), @event.GetEventInfo(
)[AppAttemptMetricsConstants.DiagnosticsInfoEventInfo]);
NUnit.Framework.Assert.AreEqual(appAttempt.GetTrackingUrl(), @event.GetEventInfo(
)[AppAttemptMetricsConstants.TrackingUrlEventInfo]);
NUnit.Framework.Assert.AreEqual(appAttempt.GetOriginalTrackingUrl(), @event.GetEventInfo
()[AppAttemptMetricsConstants.OriginalTrackingUrlEventInfo]);
NUnit.Framework.Assert.AreEqual(FinalApplicationStatus.Undefined.ToString(), @event
.GetEventInfo()[AppAttemptMetricsConstants.FinalStatusEventInfo]);
NUnit.Framework.Assert.AreEqual(YarnApplicationAttemptState.Finished.ToString(),
@event.GetEventInfo()[AppAttemptMetricsConstants.StateEventInfo]);
}
}
}
NUnit.Framework.Assert.IsTrue(hasRegisteredEvent && hasFinishedEvent);
}
/// <exception cref="System.IO.IOException"/>
public virtual bool Check(TimelineEntity entity)
{
try
{
return(this._enclosing.timelineACLsManager.CheckAccess(this.ugi, ApplicationAccessType
.ViewApp, entity));
}
catch (YarnException e)
{
TimelineDataManager.Log.Info("Error when verifying access for user " + this.ugi +
" on the events of the timeline entity " + new EntityIdentifier(entity.GetEntityId
(), entity.GetEntityType()), e);
return(false);
}
}