/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual IDictionary <ApplicationId, ApplicationReport> GetApplications(long
appsNum)
{
TimelineEntities entities = timelineDataManager.GetEntities(ApplicationMetricsConstants
.EntityType, null, null, null, null, null, null, appsNum == long.MaxValue ? this
.maxLoadedApplications : appsNum, EnumSet.AllOf <TimelineReader.Field>(), UserGroupInformation
.GetLoginUser());
IDictionary <ApplicationId, ApplicationReport> apps = new LinkedHashMap <ApplicationId
, ApplicationReport>();
if (entities != null && entities.GetEntities() != null)
{
foreach (TimelineEntity entity in entities.GetEntities())
{
try
{
ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt app = GenerateApplicationReport
(entity, ApplicationHistoryManagerOnTimelineStore.ApplicationReportField.All);
apps[app.appReport.GetApplicationId()] = app.appReport;
}
catch (Exception e)
{
Log.Error("Error on generating application report for " + entity.GetEntityId(), e
);
}
}
}
return(apps);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual IDictionary <ContainerId, ContainerReport> GetContainers(ApplicationAttemptId
appAttemptId)
{
ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt app = GetApplication
(appAttemptId.GetApplicationId(), ApplicationHistoryManagerOnTimelineStore.ApplicationReportField
.UserAndAcls);
CheckAccess(app);
TimelineEntities entities = timelineDataManager.GetEntities(ContainerMetricsConstants
.EntityType, new NameValuePair(ContainerMetricsConstants.ParentPrimariyFilter, appAttemptId
.ToString()), null, null, null, null, null, long.MaxValue, EnumSet.AllOf <TimelineReader.Field
>(), UserGroupInformation.GetLoginUser());
IDictionary <ContainerId, ContainerReport> containers = new LinkedHashMap <ContainerId
, ContainerReport>();
if (entities != null && entities.GetEntities() != null)
{
foreach (TimelineEntity entity in entities.GetEntities())
{
ContainerReport container = ConvertToContainerReport(entity, serverHttpAddress, app
.appReport.GetUser());
containers[container.GetContainerId()] = container;
}
}
return(containers);
}
private async void TweetList_ItemAppearing(object sender, ItemVisibilityEventArgs e)
{
if (TimelineEntities.Last() == e.Item as SimpleTimelineEntity)
{
await AddTweets(TimelineEntities.Last().Id);
}
}
/// <exception cref="System.Exception"/>
private static void PrepareTimelineStore(TimelineStore store, int scale)
{
for (int i = 1; i <= scale; ++i)
{
TimelineEntities entities = new TimelineEntities();
ApplicationId appId = ApplicationId.NewInstance(0, i);
if (i == 2)
{
entities.AddEntity(CreateApplicationTimelineEntity(appId, true, false, false));
}
else
{
entities.AddEntity(CreateApplicationTimelineEntity(appId, false, false, false));
}
store.Put(entities);
for (int j = 1; j <= scale; ++j)
{
entities = new TimelineEntities();
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, j);
entities.AddEntity(CreateAppAttemptTimelineEntity(appAttemptId));
store.Put(entities);
for (int k = 1; k <= scale; ++k)
{
entities = new TimelineEntities();
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, k);
entities.AddEntity(CreateContainerEntity(containerId));
store.Put(entities);
}
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public override TimelinePutResponse PutEntities(params TimelineEntity[] entities)
{
TimelineEntities entitiesContainer = new TimelineEntities();
entitiesContainer.AddEntities(Arrays.AsList(entities));
ClientResponse resp = DoPosting(entitiesContainer, null);
return(resp.GetEntity <TimelinePutResponse>());
}
public static void PrepareStore()
{
store = CreateStore(Scale);
TimelineEntities entities = new TimelineEntities();
entities.AddEntity(CreateApplicationTimelineEntity(ApplicationId.NewInstance(0, Scale
+ 1), true, true, false));
entities.AddEntity(CreateApplicationTimelineEntity(ApplicationId.NewInstance(0, Scale
+ 2), true, false, true));
store.Put(entities);
}
public virtual void TestMRTimelineEventHandling()
{
Configuration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
conf.SetBoolean(MRJobConfig.MapreduceJobEmitTimelineData, true);
MiniMRYarnCluster cluster = null;
try
{
cluster = new MiniMRYarnCluster(typeof(TestJobHistoryEventHandler).Name, 1);
cluster.Init(conf);
cluster.Start();
conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname
() + ":" + cluster.GetApplicationHistoryServer().GetPort());
TimelineStore ts = cluster.GetApplicationHistoryServer().GetTimelineStore();
Path inDir = new Path("input");
Path outDir = new Path("output");
RunningJob job = UtilsForTests.RunJobSucceed(new JobConf(conf), inDir, outDir);
NUnit.Framework.Assert.AreEqual(JobStatus.Succeeded, job.GetJobStatus().GetState(
).GetValue());
TimelineEntities entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null
, null, null, null, null, null);
NUnit.Framework.Assert.AreEqual(1, entities.GetEntities().Count);
TimelineEntity tEntity = entities.GetEntities()[0];
NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId());
NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType());
NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents
()[tEntity.GetEvents().Count - 1].GetEventType());
NUnit.Framework.Assert.AreEqual(EventType.JobFinished.ToString(), tEntity.GetEvents
()[0].GetEventType());
job = UtilsForTests.RunJobFail(new JobConf(conf), inDir, outDir);
NUnit.Framework.Assert.AreEqual(JobStatus.Failed, job.GetJobStatus().GetState().GetValue
());
entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null
, null, null);
NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count);
tEntity = entities.GetEntities()[0];
NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId());
NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType());
NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents
()[tEntity.GetEvents().Count - 1].GetEventType());
NUnit.Framework.Assert.AreEqual(EventType.JobFailed.ToString(), tEntity.GetEvents
()[0].GetEventType());
}
finally
{
if (cluster != null)
{
cluster.Stop();
}
}
}
/// <summary>Get the timeline entities that the given user have access to.</summary>
/// <remarks>
/// Get the timeline entities that the given user have access to. The meaning
/// of each argument has been documented with
/// <see cref="TimelineReader.GetEntities(string, long, long, long, string, long, NameValuePair, System.Collections.Generic.ICollection{E}, Sharpen.EnumSet{E}, CheckAcl)
/// "/>
/// .
/// </remarks>
/// <seealso cref="TimelineReader.GetEntities(string, long, long, long, string, long, NameValuePair, System.Collections.Generic.ICollection{E}, Sharpen.EnumSet{E}, CheckAcl)
/// "/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual TimelineEntities GetEntities(string entityType, NameValuePair primaryFilter
, ICollection <NameValuePair> secondaryFilter, long windowStart, long windowEnd,
string fromId, long fromTs, long limit, EnumSet <TimelineReader.Field> fields, UserGroupInformation
callerUGI)
{
TimelineEntities entities = null;
entities = store.GetEntities(entityType, limit, windowStart, windowEnd, fromId, fromTs
, primaryFilter, secondaryFilter, fields, new TimelineDataManager.CheckAclImpl(this
, callerUGI));
if (entities == null)
{
return(new TimelineEntities());
}
return(entities);
}
public virtual void TestUpdatingOldEntityWithoutDomainId()
{
// Set the domain to the default domain when updating
TimelineEntity entity = new TimelineEntity();
entity.SetEntityType("OLD_ENTITY_TYPE_1");
entity.SetEntityId("OLD_ENTITY_ID_1");
entity.SetDomainId(TimelineDataManager.DefaultDomainId);
entity.AddOtherInfo("NEW_OTHER_INFO_KEY", "NEW_OTHER_INFO_VALUE");
TimelineEntities entities = new TimelineEntities();
entities.AddEntity(entity);
TimelinePutResponse response = dataManaer.PostEntities(entities, UserGroupInformation
.GetCurrentUser());
NUnit.Framework.Assert.AreEqual(0, response.GetErrors().Count);
entity = store.GetEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null);
NUnit.Framework.Assert.IsNotNull(entity);
// Even in leveldb, the domain is updated to the default domain Id
NUnit.Framework.Assert.AreEqual(TimelineDataManager.DefaultDomainId, entity.GetDomainId
());
NUnit.Framework.Assert.AreEqual(1, entity.GetOtherInfo().Count);
NUnit.Framework.Assert.AreEqual("NEW_OTHER_INFO_KEY", entity.GetOtherInfo().Keys.
GetEnumerator().Next());
NUnit.Framework.Assert.AreEqual("NEW_OTHER_INFO_VALUE", entity.GetOtherInfo().Values
.GetEnumerator().Next());
// Set the domain to the non-default domain when updating
entity = new TimelineEntity();
entity.SetEntityType("OLD_ENTITY_TYPE_1");
entity.SetEntityId("OLD_ENTITY_ID_2");
entity.SetDomainId("NON_DEFAULT");
entity.AddOtherInfo("NEW_OTHER_INFO_KEY", "NEW_OTHER_INFO_VALUE");
entities = new TimelineEntities();
entities.AddEntity(entity);
response = dataManaer.PostEntities(entities, UserGroupInformation.GetCurrentUser(
));
NUnit.Framework.Assert.AreEqual(1, response.GetErrors().Count);
NUnit.Framework.Assert.AreEqual(TimelinePutResponse.TimelinePutError.AccessDenied
, response.GetErrors()[0].GetErrorCode());
entity = store.GetEntity("OLD_ENTITY_ID_2", "OLD_ENTITY_TYPE_1", null);
NUnit.Framework.Assert.IsNotNull(entity);
// In leveldb, the domain Id is still null
NUnit.Framework.Assert.IsNull(entity.GetDomainId());
// Updating is not executed
NUnit.Framework.Assert.AreEqual(0, entity.GetOtherInfo().Count);
}
public virtual void TestRelatingToOldEntityWithoutDomainId()
{
// New entity is put in the default domain
TimelineEntity entityToStore = new TimelineEntity();
entityToStore.SetEntityType("NEW_ENTITY_TYPE_1");
entityToStore.SetEntityId("NEW_ENTITY_ID_1");
entityToStore.SetDomainId(TimelineDataManager.DefaultDomainId);
entityToStore.AddRelatedEntity("OLD_ENTITY_TYPE_1", "OLD_ENTITY_ID_1");
TimelineEntities entities = new TimelineEntities();
entities.AddEntity(entityToStore);
store.Put(entities);
TimelineEntity entityToGet = store.GetEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1"
, null);
NUnit.Framework.Assert.IsNotNull(entityToGet);
NUnit.Framework.Assert.IsNull(entityToGet.GetDomainId());
NUnit.Framework.Assert.AreEqual("NEW_ENTITY_TYPE_1", entityToGet.GetRelatedEntities
().Keys.GetEnumerator().Next());
NUnit.Framework.Assert.AreEqual("NEW_ENTITY_ID_1", entityToGet.GetRelatedEntities
().Values.GetEnumerator().Next().GetEnumerator().Next());
// New entity is not put in the default domain
entityToStore = new TimelineEntity();
entityToStore.SetEntityType("NEW_ENTITY_TYPE_2");
entityToStore.SetEntityId("NEW_ENTITY_ID_2");
entityToStore.SetDomainId("NON_DEFAULT");
entityToStore.AddRelatedEntity("OLD_ENTITY_TYPE_1", "OLD_ENTITY_ID_1");
entities = new TimelineEntities();
entities.AddEntity(entityToStore);
TimelinePutResponse response = store.Put(entities);
NUnit.Framework.Assert.AreEqual(1, response.GetErrors().Count);
NUnit.Framework.Assert.AreEqual(TimelinePutResponse.TimelinePutError.ForbiddenRelation
, response.GetErrors()[0].GetErrorCode());
entityToGet = store.GetEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null);
NUnit.Framework.Assert.IsNotNull(entityToGet);
NUnit.Framework.Assert.IsNull(entityToGet.GetDomainId());
// Still have one related entity
NUnit.Framework.Assert.AreEqual(1, entityToGet.GetRelatedEntities().Keys.Count);
NUnit.Framework.Assert.AreEqual(1, entityToGet.GetRelatedEntities().Values.GetEnumerator
().Next().Count);
}
public virtual void TestGetOldEntitiesWithOutDomainId()
{
TimelineEntities entities = dataManaer.GetEntities("OLD_ENTITY_TYPE_1", null, null
, null, null, null, null, null, null, UserGroupInformation.GetCurrentUser());
NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count);
NUnit.Framework.Assert.AreEqual("OLD_ENTITY_ID_2", entities.GetEntities()[0].GetEntityId
());
NUnit.Framework.Assert.AreEqual("OLD_ENTITY_TYPE_1", entities.GetEntities()[0].GetEntityType
());
NUnit.Framework.Assert.AreEqual(TimelineDataManager.DefaultDomainId, entities.GetEntities
()[0].GetDomainId());
NUnit.Framework.Assert.AreEqual("OLD_ENTITY_ID_1", entities.GetEntities()[1].GetEntityId
());
NUnit.Framework.Assert.AreEqual("OLD_ENTITY_TYPE_1", entities.GetEntities()[1].GetEntityType
());
NUnit.Framework.Assert.AreEqual(TimelineDataManager.DefaultDomainId, entities.GetEntities
()[1].GetDomainId());
}
public virtual void TestGetEntitiesAclEnabled()
{
AdminACLsManager oldAdminACLsManager = aclsManager.SetAdminACLsManager(adminACLsManager
);
try
{
TimelineEntities entities = dataManaer.GetEntities("ACL_ENTITY_TYPE_1", null, null
, null, null, null, null, 1l, null, UserGroupInformation.CreateUserForTesting("owner_1"
, new string[] { "group1" }));
NUnit.Framework.Assert.AreEqual(1, entities.GetEntities().Count);
NUnit.Framework.Assert.AreEqual("ACL_ENTITY_ID_11", entities.GetEntities()[0].GetEntityId
());
}
finally
{
aclsManager.SetAdminACLsManager(oldAdminACLsManager);
}
}
public virtual void TestDeleteEntitiesPrimaryFilters()
{
IDictionary <string, ICollection <object> > primaryFilter = Sharpen.Collections.SingletonMap
("user", Sharpen.Collections.Singleton((object)"otheruser"));
TimelineEntities atsEntities = new TimelineEntities();
atsEntities.SetEntities(Sharpen.Collections.SingletonList(CreateEntity(entityId1b
, entityType1, 789l, Sharpen.Collections.SingletonList(ev2), null, primaryFilter
, null, domainId1)));
TimelinePutResponse response = store.Put(atsEntities);
NUnit.Framework.Assert.AreEqual(0, response.GetErrors().Count);
NameValuePair pfPair = new NameValuePair("user", "otheruser");
IList <TimelineEntity> entities = GetEntitiesWithPrimaryFilter("type_1", pfPair);
NUnit.Framework.Assert.AreEqual(1, entities.Count);
VerifyEntityInfo(entityId1b, entityType1, Sharpen.Collections.SingletonList(ev2),
EmptyRelEntities, primaryFilter, EmptyMap, entities[0], domainId1);
entities = GetEntitiesWithPrimaryFilter("type_1", userFilter);
NUnit.Framework.Assert.AreEqual(3, entities.Count);
VerifyEntityInfo(entityId1, entityType1, events1, EmptyRelEntities, primaryFilters
, otherInfo, entities[0], domainId1);
VerifyEntityInfo(entityId1b, entityType1, events1, EmptyRelEntities, primaryFilters
, otherInfo, entities[1], domainId1);
VerifyEntityInfo(entityId6, entityType1, EmptyEvents, EmptyRelEntities, primaryFilters
, otherInfo, entities[2], domainId2);
((LeveldbTimelineStore)store).DiscardOldEntities(-123l);
NUnit.Framework.Assert.AreEqual(1, GetEntitiesWithPrimaryFilter("type_1", pfPair)
.Count);
NUnit.Framework.Assert.AreEqual(3, GetEntitiesWithPrimaryFilter("type_1", userFilter
).Count);
((LeveldbTimelineStore)store).DiscardOldEntities(123l);
NUnit.Framework.Assert.AreEqual(0, GetEntities("type_1").Count);
NUnit.Framework.Assert.AreEqual(0, GetEntities("type_2").Count);
NUnit.Framework.Assert.AreEqual(0, ((LeveldbTimelineStore)store).GetEntityTypes()
.Count);
NUnit.Framework.Assert.AreEqual(0, GetEntitiesWithPrimaryFilter("type_1", pfPair)
.Count);
NUnit.Framework.Assert.AreEqual(0, GetEntitiesWithPrimaryFilter("type_1", userFilter
).Count);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual IDictionary <ApplicationAttemptId, ApplicationAttemptReport> GetApplicationAttempts
(ApplicationId appId)
{
ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt app = GetApplication
(appId, ApplicationHistoryManagerOnTimelineStore.ApplicationReportField.UserAndAcls
);
CheckAccess(app);
TimelineEntities entities = timelineDataManager.GetEntities(AppAttemptMetricsConstants
.EntityType, new NameValuePair(AppAttemptMetricsConstants.ParentPrimaryFilter, appId
.ToString()), null, null, null, null, null, long.MaxValue, EnumSet.AllOf <TimelineReader.Field
>(), UserGroupInformation.GetLoginUser());
IDictionary <ApplicationAttemptId, ApplicationAttemptReport> appAttempts = new LinkedHashMap
<ApplicationAttemptId, ApplicationAttemptReport>();
foreach (TimelineEntity entity in entities.GetEntities())
{
ApplicationAttemptReport appAttempt = ConvertToApplicationAttemptReport(entity);
appAttempts[appAttempt.GetApplicationAttemptId()] = appAttempt;
}
return(appAttempts);
}
public virtual void TestRelatingToNonExistingEntity()
{
TimelineEntity entityToStore = new TimelineEntity();
entityToStore.SetEntityType("TEST_ENTITY_TYPE_1");
entityToStore.SetEntityId("TEST_ENTITY_ID_1");
entityToStore.SetDomainId(TimelineDataManager.DefaultDomainId);
entityToStore.AddRelatedEntity("TEST_ENTITY_TYPE_2", "TEST_ENTITY_ID_2");
TimelineEntities entities = new TimelineEntities();
entities.AddEntity(entityToStore);
store.Put(entities);
TimelineEntity entityToGet = store.GetEntity("TEST_ENTITY_ID_2", "TEST_ENTITY_TYPE_2"
, null);
NUnit.Framework.Assert.IsNotNull(entityToGet);
NUnit.Framework.Assert.AreEqual("DEFAULT", entityToGet.GetDomainId());
NUnit.Framework.Assert.AreEqual("TEST_ENTITY_TYPE_1", entityToGet.GetRelatedEntities
().Keys.GetEnumerator().Next());
NUnit.Framework.Assert.AreEqual("TEST_ENTITY_ID_1", entityToGet.GetRelatedEntities
().Values.GetEnumerator().Next().GetEnumerator().Next());
}
public virtual TimelinePutResponse PostEntities(HttpServletRequest req, HttpServletResponse
res, TimelineEntities entities)
{
/* , MediaType.APPLICATION_XML */
Init(res);
UserGroupInformation callerUGI = GetUser(req);
if (callerUGI == null)
{
string msg = "The owner of the posted timeline entities is not set";
Log.Error(msg);
throw new ForbiddenException(msg);
}
try
{
return(timelineDataManager.PostEntities(entities, callerUGI));
}
catch (Exception e)
{
Log.Error("Error putting entities", e);
throw new WebApplicationException(e, Response.Status.InternalServerError);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestDSShell(bool haveDomain)
{
string[] args = new string[] { "--jar", AppmasterJar, "--num_containers", "2", "--shell_command"
, Shell.Windows ? "dir" : "ls", "--master_memory", "512", "--master_vcores", "2"
, "--container_memory", "128", "--container_vcores", "1" };
if (haveDomain)
{
string[] domainArgs = new string[] { "--domain", "TEST_DOMAIN", "--view_acls", "reader_user reader_group"
, "--modify_acls", "writer_user writer_group", "--create" };
IList <string> argsList = new AList <string>(Arrays.AsList(args));
Sharpen.Collections.AddAll(argsList, Arrays.AsList(domainArgs));
args = Sharpen.Collections.ToArray(argsList, new string[argsList.Count]);
}
Log.Info("Initializing DS Client");
Client client = new Client(new Configuration(yarnCluster.GetConfig()));
bool initSuccess = client.Init(args);
NUnit.Framework.Assert.IsTrue(initSuccess);
Log.Info("Running DS Client");
AtomicBoolean result = new AtomicBoolean(false);
Sharpen.Thread t = new _Thread_194(result, client);
t.Start();
YarnClient yarnClient = YarnClient.CreateYarnClient();
yarnClient.Init(new Configuration(yarnCluster.GetConfig()));
yarnClient.Start();
string hostName = NetUtils.GetHostname();
bool verified = false;
string errorMessage = string.Empty;
while (!verified)
{
IList <ApplicationReport> apps = yarnClient.GetApplications();
if (apps.Count == 0)
{
Sharpen.Thread.Sleep(10);
continue;
}
ApplicationReport appReport = apps[0];
if (appReport.GetHost().Equals("N/A"))
{
Sharpen.Thread.Sleep(10);
continue;
}
errorMessage = "Expected host name to start with '" + hostName + "', was '" + appReport
.GetHost() + "'. Expected rpc port to be '-1', was '" + appReport.GetRpcPort() +
"'.";
if (CheckHostname(appReport.GetHost()) && appReport.GetRpcPort() == -1)
{
verified = true;
}
if (appReport.GetYarnApplicationState() == YarnApplicationState.Finished)
{
break;
}
}
NUnit.Framework.Assert.IsTrue(errorMessage, verified);
t.Join();
Log.Info("Client run completed. Result=" + result);
NUnit.Framework.Assert.IsTrue(result.Get());
TimelineDomain domain = null;
if (haveDomain)
{
domain = yarnCluster.GetApplicationHistoryServer().GetTimelineStore().GetDomain("TEST_DOMAIN"
);
NUnit.Framework.Assert.IsNotNull(domain);
NUnit.Framework.Assert.AreEqual("reader_user reader_group", domain.GetReaders());
NUnit.Framework.Assert.AreEqual("writer_user writer_group", domain.GetWriters());
}
TimelineEntities entitiesAttempts = yarnCluster.GetApplicationHistoryServer().GetTimelineStore
().GetEntities(ApplicationMaster.DSEntity.DsAppAttempt.ToString(), null, null, null
, null, null, null, null, null, null);
NUnit.Framework.Assert.IsNotNull(entitiesAttempts);
NUnit.Framework.Assert.AreEqual(1, entitiesAttempts.GetEntities().Count);
NUnit.Framework.Assert.AreEqual(2, entitiesAttempts.GetEntities()[0].GetEvents().
Count);
NUnit.Framework.Assert.AreEqual(entitiesAttempts.GetEntities()[0].GetEntityType()
.ToString(), ApplicationMaster.DSEntity.DsAppAttempt.ToString());
if (haveDomain)
{
NUnit.Framework.Assert.AreEqual(domain.GetId(), entitiesAttempts.GetEntities()[0]
.GetDomainId());
}
else
{
NUnit.Framework.Assert.AreEqual("DEFAULT", entitiesAttempts.GetEntities()[0].GetDomainId
());
}
TimelineEntities entities = yarnCluster.GetApplicationHistoryServer().GetTimelineStore
().GetEntities(ApplicationMaster.DSEntity.DsContainer.ToString(), null, null, null
, null, null, null, null, null, null);
NUnit.Framework.Assert.IsNotNull(entities);
NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count);
NUnit.Framework.Assert.AreEqual(entities.GetEntities()[0].GetEntityType().ToString
(), ApplicationMaster.DSEntity.DsContainer.ToString());
if (haveDomain)
{
NUnit.Framework.Assert.AreEqual(domain.GetId(), entities.GetEntities()[0].GetDomainId
());
}
else
{
NUnit.Framework.Assert.AreEqual("DEFAULT", entities.GetEntities()[0].GetDomainId(
));
}
}
/// <exception cref="System.IO.IOException"/>
public virtual TimelineEntities GetEntities(string entityType, long limit, long windowStart
, long windowEnd, string fromId, long fromTs, NameValuePair primaryFilter, ICollection
<NameValuePair> secondaryFilters, EnumSet <TimelineReader.Field> fields, TimelineDataManager.CheckAcl
checkAcl)
{
lock (this)
{
if (limit == null)
{
limit = DefaultLimit;
}
if (windowStart == null)
{
windowStart = long.MinValue;
}
if (windowEnd == null)
{
windowEnd = long.MaxValue;
}
if (fields == null)
{
fields = EnumSet.AllOf <TimelineReader.Field>();
}
IEnumerator <TimelineEntity> entityIterator = null;
if (fromId != null)
{
TimelineEntity firstEntity = entities[new EntityIdentifier(fromId, entityType)];
if (firstEntity == null)
{
return(new TimelineEntities());
}
else
{
entityIterator = new TreeSet <TimelineEntity>(entities.Values).TailSet(firstEntity
, true).GetEnumerator();
}
}
if (entityIterator == null)
{
entityIterator = new PriorityQueue <TimelineEntity>(entities.Values).GetEnumerator
();
}
IList <TimelineEntity> entitiesSelected = new AList <TimelineEntity>();
while (entityIterator.HasNext())
{
TimelineEntity entity = entityIterator.Next();
if (entitiesSelected.Count >= limit)
{
break;
}
if (!entity.GetEntityType().Equals(entityType))
{
continue;
}
if (entity.GetStartTime() <= windowStart)
{
continue;
}
if (entity.GetStartTime() > windowEnd)
{
continue;
}
if (fromTs != null && entityInsertTimes[new EntityIdentifier(entity.GetEntityId()
, entity.GetEntityType())] > fromTs)
{
continue;
}
if (primaryFilter != null && !MatchPrimaryFilter(entity.GetPrimaryFilters(), primaryFilter
))
{
continue;
}
if (secondaryFilters != null)
{
// AND logic
bool flag = true;
foreach (NameValuePair secondaryFilter in secondaryFilters)
{
if (secondaryFilter != null && !MatchPrimaryFilter(entity.GetPrimaryFilters(), secondaryFilter
) && !MatchFilter(entity.GetOtherInfo(), secondaryFilter))
{
flag = false;
break;
}
}
if (!flag)
{
continue;
}
}
if (entity.GetDomainId() == null)
{
entity.SetDomainId(TimelineDataManager.DefaultDomainId);
}
if (checkAcl == null || checkAcl.Check(entity))
{
entitiesSelected.AddItem(entity);
}
}
IList <TimelineEntity> entitiesToReturn = new AList <TimelineEntity>();
foreach (TimelineEntity entitySelected in entitiesSelected)
{
entitiesToReturn.AddItem(MaskFields(entitySelected, fields));
}
entitiesToReturn.Sort();
TimelineEntities entitiesWrapper = new TimelineEntities();
entitiesWrapper.SetEntities(entitiesToReturn);
return(entitiesWrapper);
}
}
public virtual TimelinePutResponse Put(TimelineEntities data)
{
lock (this)
{
TimelinePutResponse response = new TimelinePutResponse();
foreach (TimelineEntity entity in data.GetEntities())
{
EntityIdentifier entityId = new EntityIdentifier(entity.GetEntityId(), entity.GetEntityType
());
// store entity info in memory
TimelineEntity existingEntity = entities[entityId];
if (existingEntity == null)
{
existingEntity = new TimelineEntity();
existingEntity.SetEntityId(entity.GetEntityId());
existingEntity.SetEntityType(entity.GetEntityType());
existingEntity.SetStartTime(entity.GetStartTime());
if (entity.GetDomainId() == null || entity.GetDomainId().Length == 0)
{
TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError
();
error.SetEntityId(entityId.GetId());
error.SetEntityType(entityId.GetType());
error.SetErrorCode(TimelinePutResponse.TimelinePutError.NoDomain);
response.AddError(error);
continue;
}
existingEntity.SetDomainId(entity.GetDomainId());
entities[entityId] = existingEntity;
entityInsertTimes[entityId] = Runtime.CurrentTimeMillis();
}
if (entity.GetEvents() != null)
{
if (existingEntity.GetEvents() == null)
{
existingEntity.SetEvents(entity.GetEvents());
}
else
{
existingEntity.AddEvents(entity.GetEvents());
}
existingEntity.GetEvents().Sort();
}
// check startTime
if (existingEntity.GetStartTime() == null)
{
if (existingEntity.GetEvents() == null || existingEntity.GetEvents().IsEmpty())
{
TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError
();
error.SetEntityId(entityId.GetId());
error.SetEntityType(entityId.GetType());
error.SetErrorCode(TimelinePutResponse.TimelinePutError.NoStartTime);
response.AddError(error);
Sharpen.Collections.Remove(entities, entityId);
Sharpen.Collections.Remove(entityInsertTimes, entityId);
continue;
}
else
{
long min = long.MaxValue;
foreach (TimelineEvent e in entity.GetEvents())
{
if (min > e.GetTimestamp())
{
min = e.GetTimestamp();
}
}
existingEntity.SetStartTime(min);
}
}
if (entity.GetPrimaryFilters() != null)
{
if (existingEntity.GetPrimaryFilters() == null)
{
existingEntity.SetPrimaryFilters(new Dictionary <string, ICollection <object> >());
}
foreach (KeyValuePair <string, ICollection <object> > pf in entity.GetPrimaryFilters
())
{
foreach (object pfo in pf.Value)
{
existingEntity.AddPrimaryFilter(pf.Key, MaybeConvert(pfo));
}
}
}
if (entity.GetOtherInfo() != null)
{
if (existingEntity.GetOtherInfo() == null)
{
existingEntity.SetOtherInfo(new Dictionary <string, object>());
}
foreach (KeyValuePair <string, object> info in entity.GetOtherInfo())
{
existingEntity.AddOtherInfo(info.Key, MaybeConvert(info.Value));
}
}
// relate it to other entities
if (entity.GetRelatedEntities() == null)
{
continue;
}
foreach (KeyValuePair <string, ICollection <string> > partRelatedEntities in entity.
GetRelatedEntities())
{
if (partRelatedEntities == null)
{
continue;
}
foreach (string idStr in partRelatedEntities.Value)
{
EntityIdentifier relatedEntityId = new EntityIdentifier(idStr, partRelatedEntities
.Key);
TimelineEntity relatedEntity = entities[relatedEntityId];
if (relatedEntity != null)
{
if (relatedEntity.GetDomainId().Equals(existingEntity.GetDomainId()))
{
relatedEntity.AddRelatedEntity(existingEntity.GetEntityType(), existingEntity.GetEntityId
());
}
else
{
// in this case the entity will be put, but the relation will be
// ignored
TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError
();
error.SetEntityType(existingEntity.GetEntityType());
error.SetEntityId(existingEntity.GetEntityId());
error.SetErrorCode(TimelinePutResponse.TimelinePutError.ForbiddenRelation);
response.AddError(error);
}
}
else
{
relatedEntity = new TimelineEntity();
relatedEntity.SetEntityId(relatedEntityId.GetId());
relatedEntity.SetEntityType(relatedEntityId.GetType());
relatedEntity.SetStartTime(existingEntity.GetStartTime());
relatedEntity.AddRelatedEntity(existingEntity.GetEntityType(), existingEntity.GetEntityId
());
relatedEntity.SetDomainId(existingEntity.GetDomainId());
entities[relatedEntityId] = relatedEntity;
entityInsertTimes[relatedEntityId] = Runtime.CurrentTimeMillis();
}
}
}
}
return(response);
}
}
private async Task AddTweets(long?maxId = null)
{
try
{
var timeline = await Tokens.Statuses.HomeTimelineAsync(count : TweetCount, max_id : maxId);
// リストビューへの表示
foreach (var item in timeline)
{
var timelineEntity = new SimpleTimelineEntity
{
Id = item.Id,
IsRetweet = item.RetweetedStatus != null
};
// リツイートかどうかで情報をわける
if (timelineEntity.IsRetweet)
{
// リツイートの場合は元ツイートの内容を格納
timelineEntity.TweetText = item.RetweetedStatus.Text;
timelineEntity.UserName = item.RetweetedStatus.User.Name;
timelineEntity.UserProfileImageUrl = item.RetweetedStatus.User.ProfileImageUrl;
timelineEntity.CreatedAt = item.RetweetedStatus.CreatedAt.LocalDateTime.ToString("F");
timelineEntity.RetweetMessage = $"{item.User.Name}さんがリツイート";
}
else
{
// それ以外はそのまま格納
timelineEntity.TweetText = item.Text;
timelineEntity.UserName = item.User.Name;
timelineEntity.UserProfileImageUrl = item.User.ProfileImageUrl;
timelineEntity.CreatedAt = item.CreatedAt.LocalDateTime.ToString("F");
};
if (item.ExtendedEntities != null)
{
var tweetImages = new List <TweetImageInfo>();
foreach (var media in item.ExtendedEntities.Media)
{
tweetImages.Add(new TweetImageInfo
{
Url = media.MediaUrl,
IsFoodPorn = await IsFoodPornAsync(media.MediaUrl)
});
}
// Twitterの画像は最大4枚なので要素数は4固定
timelineEntity.TweetImages = new TweetImageInfo[4];
Array.Copy(tweetImages.ToArray(), timelineEntity.TweetImages, tweetImages.Count());
}
TimelineEntities.Add(timelineEntity);
}
}
catch (Exception ex)
{
if (ex.Message == "Rate limit exceeded")
{
await System.Threading.Tasks.Task.Delay(1000);
await AddTweets(maxId);
}
else
{
await DisplayAlert("警告", "ツイートの読み込みに失敗しました。", "閉じる");
}
}
}
/// <summary>
/// Store the timeline entities into the store and set the owner of them to the
/// given user.
/// </summary>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual TimelinePutResponse PostEntities(TimelineEntities entities, UserGroupInformation
callerUGI)
{
if (entities == null)
{
return(new TimelinePutResponse());
}
IList <EntityIdentifier> entityIDs = new AList <EntityIdentifier>();
TimelineEntities entitiesToPut = new TimelineEntities();
IList <TimelinePutResponse.TimelinePutError> errors = new AList <TimelinePutResponse.TimelinePutError
>();
foreach (TimelineEntity entity in entities.GetEntities())
{
EntityIdentifier entityID = new EntityIdentifier(entity.GetEntityId(), entity.GetEntityType
());
// if the domain id is not specified, the entity will be put into
// the default domain
if (entity.GetDomainId() == null || entity.GetDomainId().Length == 0)
{
entity.SetDomainId(DefaultDomainId);
}
// check if there is existing entity
TimelineEntity existingEntity = null;
try
{
existingEntity = store.GetEntity(entityID.GetId(), entityID.GetType(), EnumSet.Of
(TimelineReader.Field.PrimaryFilters));
if (existingEntity != null)
{
AddDefaultDomainIdIfAbsent(existingEntity);
if (!existingEntity.GetDomainId().Equals(entity.GetDomainId()))
{
throw new YarnException("The domain of the timeline entity " + entityID + " is not allowed to be changed."
);
}
}
if (!timelineACLsManager.CheckAccess(callerUGI, ApplicationAccessType.ModifyApp,
entity))
{
throw new YarnException(callerUGI + " is not allowed to put the timeline entity "
+ entityID + " into the domain " + entity.GetDomainId() + ".");
}
}
catch (Exception e)
{
// Skip the entity which already exists and was put by others
Log.Error("Skip the timeline entity: " + entityID, e);
TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError
();
error.SetEntityId(entityID.GetId());
error.SetEntityType(entityID.GetType());
error.SetErrorCode(TimelinePutResponse.TimelinePutError.AccessDenied);
errors.AddItem(error);
continue;
}
entityIDs.AddItem(entityID);
entitiesToPut.AddEntity(entity);
if (Log.IsDebugEnabled())
{
Log.Debug("Storing the entity " + entityID + ", JSON-style content: " + TimelineUtils
.DumpTimelineRecordtoJSON(entity));
}
}
if (Log.IsDebugEnabled())
{
Log.Debug("Storing entities: " + StringHelper.CsvJoiner.Join(entityIDs));
}
TimelinePutResponse response = store.Put(entitiesToPut);
// add the errors of timeline system filter key conflict
response.AddErrors(errors);
return(response);
}
/// <summary>Put timeline data in a JSON file via command line.</summary>
/// <param name="path">path to the timeline data JSON file</param>
/// <param name="type">the type of the timeline data in the JSON file</param>
private static void PutTimelineDataInJSONFile(string path, string type)
{
FilePath jsonFile = new FilePath(path);
if (!jsonFile.Exists())
{
Log.Error("File [" + jsonFile.GetAbsolutePath() + "] doesn't exist");
return;
}
ObjectMapper mapper = new ObjectMapper();
YarnJacksonJaxbJsonProvider.ConfigObjectMapper(mapper);
TimelineEntities entities = null;
TimelineDomains domains = null;
try
{
if (type.Equals(EntityDataType))
{
entities = mapper.ReadValue <TimelineEntities>(jsonFile);
}
else
{
if (type.Equals(DomainDataType))
{
domains = mapper.ReadValue <TimelineDomains>(jsonFile);
}
}
}
catch (Exception e)
{
Log.Error("Error when reading " + e.Message);
Sharpen.Runtime.PrintStackTrace(e, System.Console.Error);
return;
}
Configuration conf = new YarnConfiguration();
TimelineClient client = TimelineClient.CreateTimelineClient();
client.Init(conf);
client.Start();
try
{
if (UserGroupInformation.IsSecurityEnabled() && conf.GetBoolean(YarnConfiguration
.TimelineServiceEnabled, false))
{
Org.Apache.Hadoop.Security.Token.Token <TimelineDelegationTokenIdentifier> token =
client.GetDelegationToken(UserGroupInformation.GetCurrentUser().GetUserName());
UserGroupInformation.GetCurrentUser().AddToken(token);
}
if (type.Equals(EntityDataType))
{
TimelinePutResponse response = client.PutEntities(Sharpen.Collections.ToArray(entities
.GetEntities(), new TimelineEntity[entities.GetEntities().Count]));
if (response.GetErrors().Count == 0)
{
Log.Info("Timeline entities are successfully put");
}
else
{
foreach (TimelinePutResponse.TimelinePutError error in response.GetErrors())
{
Log.Error("TimelineEntity [" + error.GetEntityType() + ":" + error.GetEntityId()
+ "] is not successfully put. Error code: " + error.GetErrorCode());
}
}
}
else
{
if (type.Equals(DomainDataType))
{
bool hasError = false;
foreach (TimelineDomain domain in domains.GetDomains())
{
try
{
client.PutDomain(domain);
}
catch (Exception e)
{
Log.Error("Error when putting domain " + domain.GetId(), e);
hasError = true;
}
}
if (!hasError)
{
Log.Info("Timeline domains are successfully put");
}
}
}
}
catch (RuntimeException e)
{
Log.Error("Error when putting the timeline data", e);
}
catch (Exception e)
{
Log.Error("Error when putting the timeline data", e);
}
finally
{
client.Stop();
}
}
