public virtual void TestPutEntities()
{
TestTimelineWebServicesWithSSL.TestTimelineClient client = new TestTimelineWebServicesWithSSL.TestTimelineClient
();
try
{
client.Init(conf);
client.Start();
TimelineEntity expectedEntity = new TimelineEntity();
expectedEntity.SetEntityType("test entity type");
expectedEntity.SetEntityId("test entity id");
expectedEntity.SetDomainId("test domain id");
TimelineEvent @event = new TimelineEvent();
@event.SetEventType("test event type");
@event.SetTimestamp(0L);
expectedEntity.AddEvent(@event);
TimelinePutResponse response = client.PutEntities(expectedEntity);
NUnit.Framework.Assert.AreEqual(0, response.GetErrors().Count);
NUnit.Framework.Assert.IsTrue(client.resp.ToString().Contains("https"));
TimelineEntity actualEntity = store.GetEntity(expectedEntity.GetEntityId(), expectedEntity
.GetEntityType(), EnumSet.AllOf <TimelineReader.Field>());
NUnit.Framework.Assert.IsNotNull(actualEntity);
NUnit.Framework.Assert.AreEqual(expectedEntity.GetEntityId(), actualEntity.GetEntityId
());
NUnit.Framework.Assert.AreEqual(expectedEntity.GetEntityType(), actualEntity.GetEntityType
());
}
finally
{
client.Stop();
client.Close();
}
}
private static ClientResponse MockEntityClientResponse(TimelineClientImpl client,
ClientResponse.Status status, bool hasError, bool hasRuntimeError)
{
ClientResponse response = Org.Mockito.Mockito.Mock <ClientResponse>();
if (hasRuntimeError)
{
Org.Mockito.Mockito.DoThrow(new ClientHandlerException(new ConnectException())).When
(client).DoPostingObject(Matchers.Any <TimelineEntities>(), Matchers.Any <string>(
));
return(response);
}
Org.Mockito.Mockito.DoReturn(response).When(client).DoPostingObject(Matchers.Any <
TimelineEntities>(), Matchers.Any <string>());
Org.Mockito.Mockito.When(response.GetClientResponseStatus()).ThenReturn(status);
TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError
();
error.SetEntityId("test entity id");
error.SetEntityType("test entity type");
error.SetErrorCode(TimelinePutResponse.TimelinePutError.IoException);
TimelinePutResponse putResponse = new TimelinePutResponse();
if (hasError)
{
putResponse.AddError(error);
}
Org.Mockito.Mockito.When(response.GetEntity <TimelinePutResponse>()).ThenReturn(putResponse
);
return(response);
}
public virtual void TestPostEntities()
{
MockEntityClientResponse(client, ClientResponse.Status.Ok, false, false);
try
{
TimelinePutResponse response = client.PutEntities(GenerateEntity());
NUnit.Framework.Assert.AreEqual(0, response.GetErrors().Count);
}
catch (YarnException)
{
NUnit.Framework.Assert.Fail("Exception is not expected");
}
}
public virtual void TestUpdatingOldEntityWithoutDomainId()
{
// Set the domain to the default domain when updating
TimelineEntity entity = new TimelineEntity();
entity.SetEntityType("OLD_ENTITY_TYPE_1");
entity.SetEntityId("OLD_ENTITY_ID_1");
entity.SetDomainId(TimelineDataManager.DefaultDomainId);
entity.AddOtherInfo("NEW_OTHER_INFO_KEY", "NEW_OTHER_INFO_VALUE");
TimelineEntities entities = new TimelineEntities();
entities.AddEntity(entity);
TimelinePutResponse response = dataManaer.PostEntities(entities, UserGroupInformation
.GetCurrentUser());
NUnit.Framework.Assert.AreEqual(0, response.GetErrors().Count);
entity = store.GetEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null);
NUnit.Framework.Assert.IsNotNull(entity);
// Even in leveldb, the domain is updated to the default domain Id
NUnit.Framework.Assert.AreEqual(TimelineDataManager.DefaultDomainId, entity.GetDomainId
());
NUnit.Framework.Assert.AreEqual(1, entity.GetOtherInfo().Count);
NUnit.Framework.Assert.AreEqual("NEW_OTHER_INFO_KEY", entity.GetOtherInfo().Keys.
GetEnumerator().Next());
NUnit.Framework.Assert.AreEqual("NEW_OTHER_INFO_VALUE", entity.GetOtherInfo().Values
.GetEnumerator().Next());
// Set the domain to the non-default domain when updating
entity = new TimelineEntity();
entity.SetEntityType("OLD_ENTITY_TYPE_1");
entity.SetEntityId("OLD_ENTITY_ID_2");
entity.SetDomainId("NON_DEFAULT");
entity.AddOtherInfo("NEW_OTHER_INFO_KEY", "NEW_OTHER_INFO_VALUE");
entities = new TimelineEntities();
entities.AddEntity(entity);
response = dataManaer.PostEntities(entities, UserGroupInformation.GetCurrentUser(
));
NUnit.Framework.Assert.AreEqual(1, response.GetErrors().Count);
NUnit.Framework.Assert.AreEqual(TimelinePutResponse.TimelinePutError.AccessDenied
, response.GetErrors()[0].GetErrorCode());
entity = store.GetEntity("OLD_ENTITY_ID_2", "OLD_ENTITY_TYPE_1", null);
NUnit.Framework.Assert.IsNotNull(entity);
// In leveldb, the domain Id is still null
NUnit.Framework.Assert.IsNull(entity.GetDomainId());
// Updating is not executed
NUnit.Framework.Assert.AreEqual(0, entity.GetOtherInfo().Count);
}
public virtual void TestRelatingToOldEntityWithoutDomainId()
{
// New entity is put in the default domain
TimelineEntity entityToStore = new TimelineEntity();
entityToStore.SetEntityType("NEW_ENTITY_TYPE_1");
entityToStore.SetEntityId("NEW_ENTITY_ID_1");
entityToStore.SetDomainId(TimelineDataManager.DefaultDomainId);
entityToStore.AddRelatedEntity("OLD_ENTITY_TYPE_1", "OLD_ENTITY_ID_1");
TimelineEntities entities = new TimelineEntities();
entities.AddEntity(entityToStore);
store.Put(entities);
TimelineEntity entityToGet = store.GetEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1"
, null);
NUnit.Framework.Assert.IsNotNull(entityToGet);
NUnit.Framework.Assert.IsNull(entityToGet.GetDomainId());
NUnit.Framework.Assert.AreEqual("NEW_ENTITY_TYPE_1", entityToGet.GetRelatedEntities
().Keys.GetEnumerator().Next());
NUnit.Framework.Assert.AreEqual("NEW_ENTITY_ID_1", entityToGet.GetRelatedEntities
().Values.GetEnumerator().Next().GetEnumerator().Next());
// New entity is not put in the default domain
entityToStore = new TimelineEntity();
entityToStore.SetEntityType("NEW_ENTITY_TYPE_2");
entityToStore.SetEntityId("NEW_ENTITY_ID_2");
entityToStore.SetDomainId("NON_DEFAULT");
entityToStore.AddRelatedEntity("OLD_ENTITY_TYPE_1", "OLD_ENTITY_ID_1");
entities = new TimelineEntities();
entities.AddEntity(entityToStore);
TimelinePutResponse response = store.Put(entities);
NUnit.Framework.Assert.AreEqual(1, response.GetErrors().Count);
NUnit.Framework.Assert.AreEqual(TimelinePutResponse.TimelinePutError.ForbiddenRelation
, response.GetErrors()[0].GetErrorCode());
entityToGet = store.GetEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null);
NUnit.Framework.Assert.IsNotNull(entityToGet);
NUnit.Framework.Assert.IsNull(entityToGet.GetDomainId());
// Still have one related entity
NUnit.Framework.Assert.AreEqual(1, entityToGet.GetRelatedEntities().Keys.Count);
NUnit.Framework.Assert.AreEqual(1, entityToGet.GetRelatedEntities().Values.GetEnumerator
().Next().Count);
}
public virtual void TestPostEntitiesWithError()
{
MockEntityClientResponse(client, ClientResponse.Status.Ok, true, false);
try
{
TimelinePutResponse response = client.PutEntities(GenerateEntity());
NUnit.Framework.Assert.AreEqual(1, response.GetErrors().Count);
NUnit.Framework.Assert.AreEqual("test entity id", response.GetErrors()[0].GetEntityId
());
NUnit.Framework.Assert.AreEqual("test entity type", response.GetErrors()[0].GetEntityType
());
NUnit.Framework.Assert.AreEqual(TimelinePutResponse.TimelinePutError.IoException,
response.GetErrors()[0].GetErrorCode());
}
catch (YarnException)
{
NUnit.Framework.Assert.Fail("Exception is not expected");
}
}
/// <exception cref="System.Exception"/>
public Void Call()
{
TimelineClient client = this._enclosing.CreateTimelineClientForUGI();
TimelineEntity entityToStore = new TimelineEntity();
entityToStore.SetEntityType(typeof(Org.Apache.Hadoop.Yarn.Server.Timeline.Security.TestTimelineAuthenticationFilter
).FullName);
entityToStore.SetEntityId("entity1");
entityToStore.SetStartTime(0L);
TimelinePutResponse putResponse = client.PutEntities(entityToStore);
NUnit.Framework.Assert.AreEqual(0, putResponse.GetErrors().Count);
TimelineEntity entityToRead = Org.Apache.Hadoop.Yarn.Server.Timeline.Security.TestTimelineAuthenticationFilter
.testTimelineServer.GetTimelineStore().GetEntity("entity1", typeof(Org.Apache.Hadoop.Yarn.Server.Timeline.Security.TestTimelineAuthenticationFilter
).FullName, null);
NUnit.Framework.Assert.IsNotNull(entityToRead);
return(null);
}
public virtual void TestDeleteEntitiesPrimaryFilters()
{
IDictionary <string, ICollection <object> > primaryFilter = Sharpen.Collections.SingletonMap
("user", Sharpen.Collections.Singleton((object)"otheruser"));
TimelineEntities atsEntities = new TimelineEntities();
atsEntities.SetEntities(Sharpen.Collections.SingletonList(CreateEntity(entityId1b
, entityType1, 789l, Sharpen.Collections.SingletonList(ev2), null, primaryFilter
, null, domainId1)));
TimelinePutResponse response = store.Put(atsEntities);
NUnit.Framework.Assert.AreEqual(0, response.GetErrors().Count);
NameValuePair pfPair = new NameValuePair("user", "otheruser");
IList <TimelineEntity> entities = GetEntitiesWithPrimaryFilter("type_1", pfPair);
NUnit.Framework.Assert.AreEqual(1, entities.Count);
VerifyEntityInfo(entityId1b, entityType1, Sharpen.Collections.SingletonList(ev2),
EmptyRelEntities, primaryFilter, EmptyMap, entities[0], domainId1);
entities = GetEntitiesWithPrimaryFilter("type_1", userFilter);
NUnit.Framework.Assert.AreEqual(3, entities.Count);
VerifyEntityInfo(entityId1, entityType1, events1, EmptyRelEntities, primaryFilters
, otherInfo, entities[0], domainId1);
VerifyEntityInfo(entityId1b, entityType1, events1, EmptyRelEntities, primaryFilters
, otherInfo, entities[1], domainId1);
VerifyEntityInfo(entityId6, entityType1, EmptyEvents, EmptyRelEntities, primaryFilters
, otherInfo, entities[2], domainId2);
((LeveldbTimelineStore)store).DiscardOldEntities(-123l);
NUnit.Framework.Assert.AreEqual(1, GetEntitiesWithPrimaryFilter("type_1", pfPair)
.Count);
NUnit.Framework.Assert.AreEqual(3, GetEntitiesWithPrimaryFilter("type_1", userFilter
).Count);
((LeveldbTimelineStore)store).DiscardOldEntities(123l);
NUnit.Framework.Assert.AreEqual(0, GetEntities("type_1").Count);
NUnit.Framework.Assert.AreEqual(0, GetEntities("type_2").Count);
NUnit.Framework.Assert.AreEqual(0, ((LeveldbTimelineStore)store).GetEntityTypes()
.Count);
NUnit.Framework.Assert.AreEqual(0, GetEntitiesWithPrimaryFilter("type_1", pfPair)
.Count);
NUnit.Framework.Assert.AreEqual(0, GetEntitiesWithPrimaryFilter("type_1", userFilter
).Count);
}
public virtual TimelinePutResponse Put(TimelineEntities data)
{
lock (this)
{
TimelinePutResponse response = new TimelinePutResponse();
foreach (TimelineEntity entity in data.GetEntities())
{
EntityIdentifier entityId = new EntityIdentifier(entity.GetEntityId(), entity.GetEntityType
());
// store entity info in memory
TimelineEntity existingEntity = entities[entityId];
if (existingEntity == null)
{
existingEntity = new TimelineEntity();
existingEntity.SetEntityId(entity.GetEntityId());
existingEntity.SetEntityType(entity.GetEntityType());
existingEntity.SetStartTime(entity.GetStartTime());
if (entity.GetDomainId() == null || entity.GetDomainId().Length == 0)
{
TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError
();
error.SetEntityId(entityId.GetId());
error.SetEntityType(entityId.GetType());
error.SetErrorCode(TimelinePutResponse.TimelinePutError.NoDomain);
response.AddError(error);
continue;
}
existingEntity.SetDomainId(entity.GetDomainId());
entities[entityId] = existingEntity;
entityInsertTimes[entityId] = Runtime.CurrentTimeMillis();
}
if (entity.GetEvents() != null)
{
if (existingEntity.GetEvents() == null)
{
existingEntity.SetEvents(entity.GetEvents());
}
else
{
existingEntity.AddEvents(entity.GetEvents());
}
existingEntity.GetEvents().Sort();
}
// check startTime
if (existingEntity.GetStartTime() == null)
{
if (existingEntity.GetEvents() == null || existingEntity.GetEvents().IsEmpty())
{
TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError
();
error.SetEntityId(entityId.GetId());
error.SetEntityType(entityId.GetType());
error.SetErrorCode(TimelinePutResponse.TimelinePutError.NoStartTime);
response.AddError(error);
Sharpen.Collections.Remove(entities, entityId);
Sharpen.Collections.Remove(entityInsertTimes, entityId);
continue;
}
else
{
long min = long.MaxValue;
foreach (TimelineEvent e in entity.GetEvents())
{
if (min > e.GetTimestamp())
{
min = e.GetTimestamp();
}
}
existingEntity.SetStartTime(min);
}
}
if (entity.GetPrimaryFilters() != null)
{
if (existingEntity.GetPrimaryFilters() == null)
{
existingEntity.SetPrimaryFilters(new Dictionary <string, ICollection <object> >());
}
foreach (KeyValuePair <string, ICollection <object> > pf in entity.GetPrimaryFilters
())
{
foreach (object pfo in pf.Value)
{
existingEntity.AddPrimaryFilter(pf.Key, MaybeConvert(pfo));
}
}
}
if (entity.GetOtherInfo() != null)
{
if (existingEntity.GetOtherInfo() == null)
{
existingEntity.SetOtherInfo(new Dictionary <string, object>());
}
foreach (KeyValuePair <string, object> info in entity.GetOtherInfo())
{
existingEntity.AddOtherInfo(info.Key, MaybeConvert(info.Value));
}
}
// relate it to other entities
if (entity.GetRelatedEntities() == null)
{
continue;
}
foreach (KeyValuePair <string, ICollection <string> > partRelatedEntities in entity.
GetRelatedEntities())
{
if (partRelatedEntities == null)
{
continue;
}
foreach (string idStr in partRelatedEntities.Value)
{
EntityIdentifier relatedEntityId = new EntityIdentifier(idStr, partRelatedEntities
.Key);
TimelineEntity relatedEntity = entities[relatedEntityId];
if (relatedEntity != null)
{
if (relatedEntity.GetDomainId().Equals(existingEntity.GetDomainId()))
{
relatedEntity.AddRelatedEntity(existingEntity.GetEntityType(), existingEntity.GetEntityId
());
}
else
{
// in this case the entity will be put, but the relation will be
// ignored
TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError
();
error.SetEntityType(existingEntity.GetEntityType());
error.SetEntityId(existingEntity.GetEntityId());
error.SetErrorCode(TimelinePutResponse.TimelinePutError.ForbiddenRelation);
response.AddError(error);
}
}
else
{
relatedEntity = new TimelineEntity();
relatedEntity.SetEntityId(relatedEntityId.GetId());
relatedEntity.SetEntityType(relatedEntityId.GetType());
relatedEntity.SetStartTime(existingEntity.GetStartTime());
relatedEntity.AddRelatedEntity(existingEntity.GetEntityType(), existingEntity.GetEntityId
());
relatedEntity.SetDomainId(existingEntity.GetDomainId());
entities[relatedEntityId] = relatedEntity;
entityInsertTimes[relatedEntityId] = Runtime.CurrentTimeMillis();
}
}
}
}
return(response);
}
}
/// <summary>Put timeline data in a JSON file via command line.</summary>
/// <param name="path">path to the timeline data JSON file</param>
/// <param name="type">the type of the timeline data in the JSON file</param>
private static void PutTimelineDataInJSONFile(string path, string type)
{
FilePath jsonFile = new FilePath(path);
if (!jsonFile.Exists())
{
Log.Error("File [" + jsonFile.GetAbsolutePath() + "] doesn't exist");
return;
}
ObjectMapper mapper = new ObjectMapper();
YarnJacksonJaxbJsonProvider.ConfigObjectMapper(mapper);
TimelineEntities entities = null;
TimelineDomains domains = null;
try
{
if (type.Equals(EntityDataType))
{
entities = mapper.ReadValue <TimelineEntities>(jsonFile);
}
else
{
if (type.Equals(DomainDataType))
{
domains = mapper.ReadValue <TimelineDomains>(jsonFile);
}
}
}
catch (Exception e)
{
Log.Error("Error when reading " + e.Message);
Sharpen.Runtime.PrintStackTrace(e, System.Console.Error);
return;
}
Configuration conf = new YarnConfiguration();
TimelineClient client = TimelineClient.CreateTimelineClient();
client.Init(conf);
client.Start();
try
{
if (UserGroupInformation.IsSecurityEnabled() && conf.GetBoolean(YarnConfiguration
.TimelineServiceEnabled, false))
{
Org.Apache.Hadoop.Security.Token.Token <TimelineDelegationTokenIdentifier> token =
client.GetDelegationToken(UserGroupInformation.GetCurrentUser().GetUserName());
UserGroupInformation.GetCurrentUser().AddToken(token);
}
if (type.Equals(EntityDataType))
{
TimelinePutResponse response = client.PutEntities(Sharpen.Collections.ToArray(entities
.GetEntities(), new TimelineEntity[entities.GetEntities().Count]));
if (response.GetErrors().Count == 0)
{
Log.Info("Timeline entities are successfully put");
}
else
{
foreach (TimelinePutResponse.TimelinePutError error in response.GetErrors())
{
Log.Error("TimelineEntity [" + error.GetEntityType() + ":" + error.GetEntityId()
+ "] is not successfully put. Error code: " + error.GetErrorCode());
}
}
}
else
{
if (type.Equals(DomainDataType))
{
bool hasError = false;
foreach (TimelineDomain domain in domains.GetDomains())
{
try
{
client.PutDomain(domain);
}
catch (Exception e)
{
Log.Error("Error when putting domain " + domain.GetId(), e);
hasError = true;
}
}
if (!hasError)
{
Log.Info("Timeline domains are successfully put");
}
}
}
}
catch (RuntimeException e)
{
Log.Error("Error when putting the timeline data", e);
}
catch (Exception e)
{
Log.Error("Error when putting the timeline data", e);
}
finally
{
client.Stop();
}
}
/// <summary>
/// Store the timeline entities into the store and set the owner of them to the
/// given user.
/// </summary>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual TimelinePutResponse PostEntities(TimelineEntities entities, UserGroupInformation
callerUGI)
{
if (entities == null)
{
return(new TimelinePutResponse());
}
IList <EntityIdentifier> entityIDs = new AList <EntityIdentifier>();
TimelineEntities entitiesToPut = new TimelineEntities();
IList <TimelinePutResponse.TimelinePutError> errors = new AList <TimelinePutResponse.TimelinePutError
>();
foreach (TimelineEntity entity in entities.GetEntities())
{
EntityIdentifier entityID = new EntityIdentifier(entity.GetEntityId(), entity.GetEntityType
());
// if the domain id is not specified, the entity will be put into
// the default domain
if (entity.GetDomainId() == null || entity.GetDomainId().Length == 0)
{
entity.SetDomainId(DefaultDomainId);
}
// check if there is existing entity
TimelineEntity existingEntity = null;
try
{
existingEntity = store.GetEntity(entityID.GetId(), entityID.GetType(), EnumSet.Of
(TimelineReader.Field.PrimaryFilters));
if (existingEntity != null)
{
AddDefaultDomainIdIfAbsent(existingEntity);
if (!existingEntity.GetDomainId().Equals(entity.GetDomainId()))
{
throw new YarnException("The domain of the timeline entity " + entityID + " is not allowed to be changed."
);
}
}
if (!timelineACLsManager.CheckAccess(callerUGI, ApplicationAccessType.ModifyApp,
entity))
{
throw new YarnException(callerUGI + " is not allowed to put the timeline entity "
+ entityID + " into the domain " + entity.GetDomainId() + ".");
}
}
catch (Exception e)
{
// Skip the entity which already exists and was put by others
Log.Error("Skip the timeline entity: " + entityID, e);
TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError
();
error.SetEntityId(entityID.GetId());
error.SetEntityType(entityID.GetType());
error.SetErrorCode(TimelinePutResponse.TimelinePutError.AccessDenied);
errors.AddItem(error);
continue;
}
entityIDs.AddItem(entityID);
entitiesToPut.AddEntity(entity);
if (Log.IsDebugEnabled())
{
Log.Debug("Storing the entity " + entityID + ", JSON-style content: " + TimelineUtils
.DumpTimelineRecordtoJSON(entity));
}
}
if (Log.IsDebugEnabled())
{
Log.Debug("Storing entities: " + StringHelper.CsvJoiner.Join(entityIDs));
}
TimelinePutResponse response = store.Put(entitiesToPut);
// add the errors of timeline system filter key conflict
response.AddErrors(errors);
return(response);
}
