private static void AddDefaultDomainIdIfAbsent(TimelineEntity entity)
{
// be compatible with the timeline data created before 2.6
if (entity.GetDomainId() == null)
{
entity.SetDomainId(DefaultDomainId);
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual bool CheckAccess(UserGroupInformation callerUGI, ApplicationAccessType
applicationAccessType, TimelineEntity entity)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Verifying the access of " + (callerUGI == null ? null : callerUGI.GetShortUserName
()) + " on the timeline entity " + new EntityIdentifier(entity.GetEntityId(), entity
.GetEntityType()));
}
if (!adminAclsManager.AreACLsEnabled())
{
return(true);
}
// find domain owner and acls
TimelineACLsManager.AccessControlListExt aclExt = aclExts[entity.GetDomainId()];
if (aclExt == null)
{
aclExt = LoadDomainFromTimelineStore(entity.GetDomainId());
}
if (aclExt == null)
{
throw new YarnException("Domain information of the timeline entity " + new EntityIdentifier
(entity.GetEntityId(), entity.GetEntityType()) + " doesn't exist.");
}
string owner = aclExt.owner;
AccessControlList domainACL = aclExt.acls[applicationAccessType];
if (domainACL == null)
{
if (Log.IsDebugEnabled())
{
Log.Debug("ACL not found for access-type " + applicationAccessType + " for domain "
+ entity.GetDomainId() + " owned by " + owner + ". Using default [" + YarnConfiguration
.DefaultYarnAppAcl + "]");
}
domainACL = new AccessControlList(YarnConfiguration.DefaultYarnAppAcl);
}
if (callerUGI != null && (adminAclsManager.IsAdmin(callerUGI) || callerUGI.GetShortUserName
().Equals(owner) || domainACL.IsUserAllowed(callerUGI)))
{
return(true);
}
return(false);
}
public virtual void TestRelatingToOldEntityWithoutDomainId()
{
// New entity is put in the default domain
TimelineEntity entityToStore = new TimelineEntity();
entityToStore.SetEntityType("NEW_ENTITY_TYPE_1");
entityToStore.SetEntityId("NEW_ENTITY_ID_1");
entityToStore.SetDomainId(TimelineDataManager.DefaultDomainId);
entityToStore.AddRelatedEntity("OLD_ENTITY_TYPE_1", "OLD_ENTITY_ID_1");
TimelineEntities entities = new TimelineEntities();
entities.AddEntity(entityToStore);
store.Put(entities);
TimelineEntity entityToGet = store.GetEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1"
, null);
NUnit.Framework.Assert.IsNotNull(entityToGet);
NUnit.Framework.Assert.IsNull(entityToGet.GetDomainId());
NUnit.Framework.Assert.AreEqual("NEW_ENTITY_TYPE_1", entityToGet.GetRelatedEntities
().Keys.GetEnumerator().Next());
NUnit.Framework.Assert.AreEqual("NEW_ENTITY_ID_1", entityToGet.GetRelatedEntities
().Values.GetEnumerator().Next().GetEnumerator().Next());
// New entity is not put in the default domain
entityToStore = new TimelineEntity();
entityToStore.SetEntityType("NEW_ENTITY_TYPE_2");
entityToStore.SetEntityId("NEW_ENTITY_ID_2");
entityToStore.SetDomainId("NON_DEFAULT");
entityToStore.AddRelatedEntity("OLD_ENTITY_TYPE_1", "OLD_ENTITY_ID_1");
entities = new TimelineEntities();
entities.AddEntity(entityToStore);
TimelinePutResponse response = store.Put(entities);
NUnit.Framework.Assert.AreEqual(1, response.GetErrors().Count);
NUnit.Framework.Assert.AreEqual(TimelinePutResponse.TimelinePutError.ForbiddenRelation
, response.GetErrors()[0].GetErrorCode());
entityToGet = store.GetEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null);
NUnit.Framework.Assert.IsNotNull(entityToGet);
NUnit.Framework.Assert.IsNull(entityToGet.GetDomainId());
// Still have one related entity
NUnit.Framework.Assert.AreEqual(1, entityToGet.GetRelatedEntities().Keys.Count);
NUnit.Framework.Assert.AreEqual(1, entityToGet.GetRelatedEntities().Values.GetEnumerator
().Next().Count);
}
public virtual void TestGetOldEntityWithOutDomainId()
{
TimelineEntity entity = dataManaer.GetEntity("OLD_ENTITY_TYPE_1", "OLD_ENTITY_ID_1"
, null, UserGroupInformation.GetCurrentUser());
NUnit.Framework.Assert.IsNotNull(entity);
NUnit.Framework.Assert.AreEqual("OLD_ENTITY_ID_1", entity.GetEntityId());
NUnit.Framework.Assert.AreEqual("OLD_ENTITY_TYPE_1", entity.GetEntityType());
NUnit.Framework.Assert.AreEqual(TimelineDataManager.DefaultDomainId, entity.GetDomainId
());
}
private static TimelineEntity MaskFields(TimelineEntity entity, EnumSet <TimelineReader.Field
> fields)
{
// Conceal the fields that are not going to be exposed
TimelineEntity entityToReturn = new TimelineEntity();
entityToReturn.SetEntityId(entity.GetEntityId());
entityToReturn.SetEntityType(entity.GetEntityType());
entityToReturn.SetStartTime(entity.GetStartTime());
entityToReturn.SetDomainId(entity.GetDomainId());
// Deep copy
if (fields.Contains(TimelineReader.Field.Events))
{
entityToReturn.AddEvents(entity.GetEvents());
}
else
{
if (fields.Contains(TimelineReader.Field.LastEventOnly))
{
entityToReturn.AddEvent(entity.GetEvents()[0]);
}
else
{
entityToReturn.SetEvents(null);
}
}
if (fields.Contains(TimelineReader.Field.RelatedEntities))
{
entityToReturn.AddRelatedEntities(entity.GetRelatedEntities());
}
else
{
entityToReturn.SetRelatedEntities(null);
}
if (fields.Contains(TimelineReader.Field.PrimaryFilters))
{
entityToReturn.AddPrimaryFilters(entity.GetPrimaryFilters());
}
else
{
entityToReturn.SetPrimaryFilters(null);
}
if (fields.Contains(TimelineReader.Field.OtherInfo))
{
entityToReturn.AddOtherInfo(entity.GetOtherInfo());
}
else
{
entityToReturn.SetOtherInfo(null);
}
return(entityToReturn);
}
public virtual void TestUpdatingOldEntityWithoutDomainId()
{
// Set the domain to the default domain when updating
TimelineEntity entity = new TimelineEntity();
entity.SetEntityType("OLD_ENTITY_TYPE_1");
entity.SetEntityId("OLD_ENTITY_ID_1");
entity.SetDomainId(TimelineDataManager.DefaultDomainId);
entity.AddOtherInfo("NEW_OTHER_INFO_KEY", "NEW_OTHER_INFO_VALUE");
TimelineEntities entities = new TimelineEntities();
entities.AddEntity(entity);
TimelinePutResponse response = dataManaer.PostEntities(entities, UserGroupInformation
.GetCurrentUser());
NUnit.Framework.Assert.AreEqual(0, response.GetErrors().Count);
entity = store.GetEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null);
NUnit.Framework.Assert.IsNotNull(entity);
// Even in leveldb, the domain is updated to the default domain Id
NUnit.Framework.Assert.AreEqual(TimelineDataManager.DefaultDomainId, entity.GetDomainId
());
NUnit.Framework.Assert.AreEqual(1, entity.GetOtherInfo().Count);
NUnit.Framework.Assert.AreEqual("NEW_OTHER_INFO_KEY", entity.GetOtherInfo().Keys.
GetEnumerator().Next());
NUnit.Framework.Assert.AreEqual("NEW_OTHER_INFO_VALUE", entity.GetOtherInfo().Values
.GetEnumerator().Next());
// Set the domain to the non-default domain when updating
entity = new TimelineEntity();
entity.SetEntityType("OLD_ENTITY_TYPE_1");
entity.SetEntityId("OLD_ENTITY_ID_2");
entity.SetDomainId("NON_DEFAULT");
entity.AddOtherInfo("NEW_OTHER_INFO_KEY", "NEW_OTHER_INFO_VALUE");
entities = new TimelineEntities();
entities.AddEntity(entity);
response = dataManaer.PostEntities(entities, UserGroupInformation.GetCurrentUser(
));
NUnit.Framework.Assert.AreEqual(1, response.GetErrors().Count);
NUnit.Framework.Assert.AreEqual(TimelinePutResponse.TimelinePutError.AccessDenied
, response.GetErrors()[0].GetErrorCode());
entity = store.GetEntity("OLD_ENTITY_ID_2", "OLD_ENTITY_TYPE_1", null);
NUnit.Framework.Assert.IsNotNull(entity);
// In leveldb, the domain Id is still null
NUnit.Framework.Assert.IsNull(entity.GetDomainId());
// Updating is not executed
NUnit.Framework.Assert.AreEqual(0, entity.GetOtherInfo().Count);
}
public virtual void TestRelatingToNonExistingEntity()
{
TimelineEntity entityToStore = new TimelineEntity();
entityToStore.SetEntityType("TEST_ENTITY_TYPE_1");
entityToStore.SetEntityId("TEST_ENTITY_ID_1");
entityToStore.SetDomainId(TimelineDataManager.DefaultDomainId);
entityToStore.AddRelatedEntity("TEST_ENTITY_TYPE_2", "TEST_ENTITY_ID_2");
TimelineEntities entities = new TimelineEntities();
entities.AddEntity(entityToStore);
store.Put(entities);
TimelineEntity entityToGet = store.GetEntity("TEST_ENTITY_ID_2", "TEST_ENTITY_TYPE_2"
, null);
NUnit.Framework.Assert.IsNotNull(entityToGet);
NUnit.Framework.Assert.AreEqual("DEFAULT", entityToGet.GetDomainId());
NUnit.Framework.Assert.AreEqual("TEST_ENTITY_TYPE_1", entityToGet.GetRelatedEntities
().Keys.GetEnumerator().Next());
NUnit.Framework.Assert.AreEqual("TEST_ENTITY_ID_1", entityToGet.GetRelatedEntities
().Values.GetEnumerator().Next().GetEnumerator().Next());
}
/// <exception cref="System.IO.IOException"/>
public virtual TimelineEntities GetEntities(string entityType, long limit, long windowStart
, long windowEnd, string fromId, long fromTs, NameValuePair primaryFilter, ICollection
<NameValuePair> secondaryFilters, EnumSet <TimelineReader.Field> fields, TimelineDataManager.CheckAcl
checkAcl)
{
lock (this)
{
if (limit == null)
{
limit = DefaultLimit;
}
if (windowStart == null)
{
windowStart = long.MinValue;
}
if (windowEnd == null)
{
windowEnd = long.MaxValue;
}
if (fields == null)
{
fields = EnumSet.AllOf <TimelineReader.Field>();
}
IEnumerator <TimelineEntity> entityIterator = null;
if (fromId != null)
{
TimelineEntity firstEntity = entities[new EntityIdentifier(fromId, entityType)];
if (firstEntity == null)
{
return(new TimelineEntities());
}
else
{
entityIterator = new TreeSet <TimelineEntity>(entities.Values).TailSet(firstEntity
, true).GetEnumerator();
}
}
if (entityIterator == null)
{
entityIterator = new PriorityQueue <TimelineEntity>(entities.Values).GetEnumerator
();
}
IList <TimelineEntity> entitiesSelected = new AList <TimelineEntity>();
while (entityIterator.HasNext())
{
TimelineEntity entity = entityIterator.Next();
if (entitiesSelected.Count >= limit)
{
break;
}
if (!entity.GetEntityType().Equals(entityType))
{
continue;
}
if (entity.GetStartTime() <= windowStart)
{
continue;
}
if (entity.GetStartTime() > windowEnd)
{
continue;
}
if (fromTs != null && entityInsertTimes[new EntityIdentifier(entity.GetEntityId()
, entity.GetEntityType())] > fromTs)
{
continue;
}
if (primaryFilter != null && !MatchPrimaryFilter(entity.GetPrimaryFilters(), primaryFilter
))
{
continue;
}
if (secondaryFilters != null)
{
// AND logic
bool flag = true;
foreach (NameValuePair secondaryFilter in secondaryFilters)
{
if (secondaryFilter != null && !MatchPrimaryFilter(entity.GetPrimaryFilters(), secondaryFilter
) && !MatchFilter(entity.GetOtherInfo(), secondaryFilter))
{
flag = false;
break;
}
}
if (!flag)
{
continue;
}
}
if (entity.GetDomainId() == null)
{
entity.SetDomainId(TimelineDataManager.DefaultDomainId);
}
if (checkAcl == null || checkAcl.Check(entity))
{
entitiesSelected.AddItem(entity);
}
}
IList <TimelineEntity> entitiesToReturn = new AList <TimelineEntity>();
foreach (TimelineEntity entitySelected in entitiesSelected)
{
entitiesToReturn.AddItem(MaskFields(entitySelected, fields));
}
entitiesToReturn.Sort();
TimelineEntities entitiesWrapper = new TimelineEntities();
entitiesWrapper.SetEntities(entitiesToReturn);
return(entitiesWrapper);
}
}
public virtual TimelinePutResponse Put(TimelineEntities data)
{
lock (this)
{
TimelinePutResponse response = new TimelinePutResponse();
foreach (TimelineEntity entity in data.GetEntities())
{
EntityIdentifier entityId = new EntityIdentifier(entity.GetEntityId(), entity.GetEntityType
());
// store entity info in memory
TimelineEntity existingEntity = entities[entityId];
if (existingEntity == null)
{
existingEntity = new TimelineEntity();
existingEntity.SetEntityId(entity.GetEntityId());
existingEntity.SetEntityType(entity.GetEntityType());
existingEntity.SetStartTime(entity.GetStartTime());
if (entity.GetDomainId() == null || entity.GetDomainId().Length == 0)
{
TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError
();
error.SetEntityId(entityId.GetId());
error.SetEntityType(entityId.GetType());
error.SetErrorCode(TimelinePutResponse.TimelinePutError.NoDomain);
response.AddError(error);
continue;
}
existingEntity.SetDomainId(entity.GetDomainId());
entities[entityId] = existingEntity;
entityInsertTimes[entityId] = Runtime.CurrentTimeMillis();
}
if (entity.GetEvents() != null)
{
if (existingEntity.GetEvents() == null)
{
existingEntity.SetEvents(entity.GetEvents());
}
else
{
existingEntity.AddEvents(entity.GetEvents());
}
existingEntity.GetEvents().Sort();
}
// check startTime
if (existingEntity.GetStartTime() == null)
{
if (existingEntity.GetEvents() == null || existingEntity.GetEvents().IsEmpty())
{
TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError
();
error.SetEntityId(entityId.GetId());
error.SetEntityType(entityId.GetType());
error.SetErrorCode(TimelinePutResponse.TimelinePutError.NoStartTime);
response.AddError(error);
Sharpen.Collections.Remove(entities, entityId);
Sharpen.Collections.Remove(entityInsertTimes, entityId);
continue;
}
else
{
long min = long.MaxValue;
foreach (TimelineEvent e in entity.GetEvents())
{
if (min > e.GetTimestamp())
{
min = e.GetTimestamp();
}
}
existingEntity.SetStartTime(min);
}
}
if (entity.GetPrimaryFilters() != null)
{
if (existingEntity.GetPrimaryFilters() == null)
{
existingEntity.SetPrimaryFilters(new Dictionary <string, ICollection <object> >());
}
foreach (KeyValuePair <string, ICollection <object> > pf in entity.GetPrimaryFilters
())
{
foreach (object pfo in pf.Value)
{
existingEntity.AddPrimaryFilter(pf.Key, MaybeConvert(pfo));
}
}
}
if (entity.GetOtherInfo() != null)
{
if (existingEntity.GetOtherInfo() == null)
{
existingEntity.SetOtherInfo(new Dictionary <string, object>());
}
foreach (KeyValuePair <string, object> info in entity.GetOtherInfo())
{
existingEntity.AddOtherInfo(info.Key, MaybeConvert(info.Value));
}
}
// relate it to other entities
if (entity.GetRelatedEntities() == null)
{
continue;
}
foreach (KeyValuePair <string, ICollection <string> > partRelatedEntities in entity.
GetRelatedEntities())
{
if (partRelatedEntities == null)
{
continue;
}
foreach (string idStr in partRelatedEntities.Value)
{
EntityIdentifier relatedEntityId = new EntityIdentifier(idStr, partRelatedEntities
.Key);
TimelineEntity relatedEntity = entities[relatedEntityId];
if (relatedEntity != null)
{
if (relatedEntity.GetDomainId().Equals(existingEntity.GetDomainId()))
{
relatedEntity.AddRelatedEntity(existingEntity.GetEntityType(), existingEntity.GetEntityId
());
}
else
{
// in this case the entity will be put, but the relation will be
// ignored
TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError
();
error.SetEntityType(existingEntity.GetEntityType());
error.SetEntityId(existingEntity.GetEntityId());
error.SetErrorCode(TimelinePutResponse.TimelinePutError.ForbiddenRelation);
response.AddError(error);
}
}
else
{
relatedEntity = new TimelineEntity();
relatedEntity.SetEntityId(relatedEntityId.GetId());
relatedEntity.SetEntityType(relatedEntityId.GetType());
relatedEntity.SetStartTime(existingEntity.GetStartTime());
relatedEntity.AddRelatedEntity(existingEntity.GetEntityType(), existingEntity.GetEntityId
());
relatedEntity.SetDomainId(existingEntity.GetDomainId());
entities[relatedEntityId] = relatedEntity;
entityInsertTimes[relatedEntityId] = Runtime.CurrentTimeMillis();
}
}
}
}
return(response);
}
}
/// <summary>
/// Store the timeline entities into the store and set the owner of them to the
/// given user.
/// </summary>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual TimelinePutResponse PostEntities(TimelineEntities entities, UserGroupInformation
callerUGI)
{
if (entities == null)
{
return(new TimelinePutResponse());
}
IList <EntityIdentifier> entityIDs = new AList <EntityIdentifier>();
TimelineEntities entitiesToPut = new TimelineEntities();
IList <TimelinePutResponse.TimelinePutError> errors = new AList <TimelinePutResponse.TimelinePutError
>();
foreach (TimelineEntity entity in entities.GetEntities())
{
EntityIdentifier entityID = new EntityIdentifier(entity.GetEntityId(), entity.GetEntityType
());
// if the domain id is not specified, the entity will be put into
// the default domain
if (entity.GetDomainId() == null || entity.GetDomainId().Length == 0)
{
entity.SetDomainId(DefaultDomainId);
}
// check if there is existing entity
TimelineEntity existingEntity = null;
try
{
existingEntity = store.GetEntity(entityID.GetId(), entityID.GetType(), EnumSet.Of
(TimelineReader.Field.PrimaryFilters));
if (existingEntity != null)
{
AddDefaultDomainIdIfAbsent(existingEntity);
if (!existingEntity.GetDomainId().Equals(entity.GetDomainId()))
{
throw new YarnException("The domain of the timeline entity " + entityID + " is not allowed to be changed."
);
}
}
if (!timelineACLsManager.CheckAccess(callerUGI, ApplicationAccessType.ModifyApp,
entity))
{
throw new YarnException(callerUGI + " is not allowed to put the timeline entity "
+ entityID + " into the domain " + entity.GetDomainId() + ".");
}
}
catch (Exception e)
{
// Skip the entity which already exists and was put by others
Log.Error("Skip the timeline entity: " + entityID, e);
TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError
();
error.SetEntityId(entityID.GetId());
error.SetEntityType(entityID.GetType());
error.SetErrorCode(TimelinePutResponse.TimelinePutError.AccessDenied);
errors.AddItem(error);
continue;
}
entityIDs.AddItem(entityID);
entitiesToPut.AddEntity(entity);
if (Log.IsDebugEnabled())
{
Log.Debug("Storing the entity " + entityID + ", JSON-style content: " + TimelineUtils
.DumpTimelineRecordtoJSON(entity));
}
}
if (Log.IsDebugEnabled())
{
Log.Debug("Storing entities: " + StringHelper.CsvJoiner.Join(entityIDs));
}
TimelinePutResponse response = store.Put(entitiesToPut);
// add the errors of timeline system filter key conflict
response.AddErrors(errors);
return(response);
}
