public virtual TimelineDomain GetDomain(HttpServletRequest req, HttpServletResponse
res, string domainId)
{
/* , MediaType.APPLICATION_XML */
Init(res);
domainId = ParseStr(domainId);
if (domainId == null || domainId.Length == 0)
{
throw new BadRequestException("Domain ID is not specified.");
}
TimelineDomain domain = null;
try
{
domain = timelineDataManager.GetDomain(ParseStr(domainId), GetUser(req));
}
catch (Exception e)
{
Log.Error("Error getting domain", e);
throw new WebApplicationException(e, Response.Status.InternalServerError);
}
if (domain == null)
{
throw new NotFoundException("Timeline domain [" + domainId + "] is not found");
}
return(domain);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual bool CheckAccess(UserGroupInformation callerUGI, TimelineDomain domain
)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Verifying the access of " + (callerUGI == null ? null : callerUGI.GetShortUserName
()) + " on the timeline domain " + domain);
}
if (!adminAclsManager.AreACLsEnabled())
{
return(true);
}
string owner = domain.GetOwner();
if (owner == null || owner.Length == 0)
{
throw new YarnException("Owner information of the timeline domain " + domain.GetId
() + " is corrupted.");
}
if (callerUGI != null && (adminAclsManager.IsAdmin(callerUGI) || callerUGI.GetShortUserName
().Equals(owner)))
{
return(true);
}
return(false);
}
public virtual void ReplaceIfExist(TimelineDomain domain)
{
if (aclExts.Contains(domain.GetId()))
{
PutDomainIntoCache(domain);
}
}
public virtual TimelinePutResponse PutDomain(HttpServletRequest req, HttpServletResponse
res, TimelineDomain domain)
{
/* , MediaType.APPLICATION_XML */
Init(res);
UserGroupInformation callerUGI = GetUser(req);
if (callerUGI == null)
{
string msg = "The owner of the posted timeline domain is not set";
Log.Error(msg);
throw new ForbiddenException(msg);
}
domain.SetOwner(callerUGI.GetShortUserName());
try
{
timelineDataManager.PutDomain(domain, callerUGI);
}
catch (YarnException e)
{
// The user doesn't have the access to override the existing domain.
Log.Error(e.Message, e);
throw new ForbiddenException(e);
}
catch (IOException e)
{
Log.Error("Error putting domain", e);
throw new WebApplicationException(e, Response.Status.InternalServerError);
}
return(new TimelinePutResponse());
}
static TestTimelineACLsManager()
{
domain = new TimelineDomain();
domain.SetId("domain_id_1");
domain.SetOwner("owner");
domain.SetReaders("reader");
domain.SetWriters("writer");
}
public virtual void TestYarnACLsNotEnabledForDomain()
{
Configuration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.YarnAclEnable, false);
TimelineACLsManager timelineACLsManager = new TimelineACLsManager(conf);
TimelineDomain domain = new TimelineDomain();
domain.SetOwner("owner");
NUnit.Framework.Assert.IsTrue("Always true when ACLs are not enabled", timelineACLsManager
.CheckAccess(UserGroupInformation.CreateRemoteUser("user"), domain));
}
public static TimelineDomain GenerateDomain()
{
TimelineDomain domain = new TimelineDomain();
domain.SetId("namesapce id");
domain.SetDescription("domain description");
domain.SetOwner("domain owner");
domain.SetReaders("domain_reader");
domain.SetWriters("domain_writer");
domain.SetCreatedTime(0L);
domain.SetModifiedTime(1L);
return(domain);
}
/// <summary>Get a single domain of the particular ID.</summary>
/// <remarks>
/// Get a single domain of the particular ID. If callerUGI is not the owner
/// or the admin of the domain, null will be returned.
/// </remarks>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual TimelineDomain GetDomain(string domainId, UserGroupInformation callerUGI
)
{
TimelineDomain domain = store.GetDomain(domainId);
if (domain != null)
{
if (timelineACLsManager.CheckAccess(callerUGI, domain))
{
return(domain);
}
}
return(null);
}
/// <exception cref="System.IO.IOException"/>
public virtual TimelineDomain GetDomain(string domainId)
{
TimelineDomain domain = domainsById[domainId];
if (domain == null)
{
return(null);
}
else
{
return(CreateTimelineDomain(domain.GetId(), domain.GetDescription(), domain.GetOwner
(), domain.GetReaders(), domain.GetWriters(), domain.GetCreatedTime(), domain.GetModifiedTime
()));
}
}
private TimelineACLsManager.AccessControlListExt PutDomainIntoCache(TimelineDomain
domain)
{
IDictionary <ApplicationAccessType, AccessControlList> acls = new Dictionary <ApplicationAccessType
, AccessControlList>(2);
acls[ApplicationAccessType.ViewApp] = new AccessControlList(StringHelper.Cjoin(domain
.GetReaders()));
acls[ApplicationAccessType.ModifyApp] = new AccessControlList(StringHelper.Cjoin(
domain.GetWriters()));
TimelineACLsManager.AccessControlListExt aclExt = new TimelineACLsManager.AccessControlListExt
(domain.GetOwner(), acls);
aclExts[domain.GetId()] = aclExt;
return(aclExt);
}
private static TimelineDomain CreateTimelineDomain(string id, string description,
string owner, string readers, string writers, long createdTime, long modifiedTime
)
{
TimelineDomain domainToStore = new TimelineDomain();
domainToStore.SetId(id);
domainToStore.SetDescription(description);
domainToStore.SetOwner(owner);
domainToStore.SetReaders(readers);
domainToStore.SetWriters(writers);
domainToStore.SetCreatedTime(createdTime);
domainToStore.SetModifiedTime(modifiedTime);
return(domainToStore);
}
/// <exception cref="System.Exception"/>
public Void Call()
{
TimelineClient client = this._enclosing.CreateTimelineClientForUGI();
TimelineDomain domainToStore = new TimelineDomain();
domainToStore.SetId(typeof(Org.Apache.Hadoop.Yarn.Server.Timeline.Security.TestTimelineAuthenticationFilter
).FullName);
domainToStore.SetReaders("*");
domainToStore.SetWriters("*");
client.PutDomain(domainToStore);
TimelineDomain domainToRead = Org.Apache.Hadoop.Yarn.Server.Timeline.Security.TestTimelineAuthenticationFilter
.testTimelineServer.GetTimelineStore().GetDomain(typeof(Org.Apache.Hadoop.Yarn.Server.Timeline.Security.TestTimelineAuthenticationFilter
).FullName);
NUnit.Framework.Assert.IsNotNull(domainToRead);
return(null);
}
public virtual void TestYarnACLsEnabledForDomain()
{
Configuration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.YarnAclEnable, true);
conf.Set(YarnConfiguration.YarnAdminAcl, "admin");
TimelineACLsManager timelineACLsManager = new TimelineACLsManager(conf);
TimelineDomain domain = new TimelineDomain();
domain.SetOwner("owner");
NUnit.Framework.Assert.IsTrue("Owner should be allowed to access", timelineACLsManager
.CheckAccess(UserGroupInformation.CreateRemoteUser("owner"), domain));
NUnit.Framework.Assert.IsFalse("Other shouldn't be allowed to access", timelineACLsManager
.CheckAccess(UserGroupInformation.CreateRemoteUser("other"), domain));
NUnit.Framework.Assert.IsTrue("Admin should be allowed to access", timelineACLsManager
.CheckAccess(UserGroupInformation.CreateRemoteUser("admin"), domain));
}
/// <exception cref="System.IO.IOException"/>
private TimelineACLsManager.AccessControlListExt LoadDomainFromTimelineStore(string
domainId)
{
if (store == null)
{
return(null);
}
TimelineDomain domain = store.GetDomain(domainId);
if (domain == null)
{
return(null);
}
else
{
return(PutDomainIntoCache(domain));
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
TimelineDomain domain = store.GetDomain("DEFAULT");
// it is okay to reuse an existing domain even if it was created by another
// user of the timeline server before, because it allows everybody to access.
if (domain == null)
{
// create a default domain, which allows everybody to access and modify
// the entities in it.
domain = new TimelineDomain();
domain.SetId(DefaultDomainId);
domain.SetDescription("System Default Domain");
domain.SetOwner(UserGroupInformation.GetCurrentUser().GetShortUserName());
domain.SetReaders("*");
domain.SetWriters("*");
store.Put(domain);
}
base.ServiceInit(conf);
}
public virtual void TestCorruptedOwnerInfoForDomain()
{
Configuration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.YarnAclEnable, true);
conf.Set(YarnConfiguration.YarnAdminAcl, "owner");
TimelineACLsManager timelineACLsManager = new TimelineACLsManager(conf);
TimelineDomain domain = new TimelineDomain();
try
{
timelineACLsManager.CheckAccess(UserGroupInformation.CreateRemoteUser("owner"), domain
);
NUnit.Framework.Assert.Fail("Exception is expected");
}
catch (YarnException e)
{
NUnit.Framework.Assert.IsTrue("It's not the exact expected exception", e.Message.
Contains("is corrupted."));
}
}
/// <exception cref="System.IO.IOException"/>
public virtual TimelineDomains GetDomains(string owner)
{
IList <TimelineDomain> domains = new AList <TimelineDomain>();
ICollection <TimelineDomain> domainsOfOneOwner = domainsByOwner[owner];
if (domainsOfOneOwner == null)
{
return(new TimelineDomains());
}
foreach (TimelineDomain domain in domainsByOwner[owner])
{
TimelineDomain domainToReturn = CreateTimelineDomain(domain.GetId(), domain.GetDescription
(), domain.GetOwner(), domain.GetReaders(), domain.GetWriters(), domain.GetCreatedTime
(), domain.GetModifiedTime());
domains.AddItem(domainToReturn);
}
domains.Sort(new _IComparer_267());
TimelineDomains domainsToReturn = new TimelineDomains();
domainsToReturn.AddDomains(domains);
return(domainsToReturn);
}
private void PrepareTimelineDomain()
{
TimelineClient timelineClient = null;
if (conf.GetBoolean(YarnConfiguration.TimelineServiceEnabled, YarnConfiguration.DefaultTimelineServiceEnabled
))
{
timelineClient = TimelineClient.CreateTimelineClient();
timelineClient.Init(conf);
timelineClient.Start();
}
else
{
Log.Warn("Cannot put the domain " + domainId + " because the timeline service is not enabled"
);
return;
}
try
{
//TODO: we need to check and combine the existing timeline domain ACLs,
//but let's do it once we have client java library to query domains.
TimelineDomain domain = new TimelineDomain();
domain.SetId(domainId);
domain.SetReaders(viewACLs != null && viewACLs.Length > 0 ? viewACLs : " ");
domain.SetWriters(modifyACLs != null && modifyACLs.Length > 0 ? modifyACLs : " ");
timelineClient.PutDomain(domain);
Log.Info("Put the timeline domain: " + TimelineUtils.DumpTimelineRecordtoJSON(domain
));
}
catch (Exception e)
{
Log.Error("Error when putting the timeline domain", e);
}
finally
{
timelineClient.Stop();
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void Put(TimelineDomain domain)
{
TimelineDomain domainToReplace = domainsById[domain.GetId()];
long currentTimestamp = Runtime.CurrentTimeMillis();
TimelineDomain domainToStore = CreateTimelineDomain(domain.GetId(), domain.GetDescription
(), domain.GetOwner(), domain.GetReaders(), domain.GetWriters(), (domainToReplace
== null ? currentTimestamp : domainToReplace.GetCreatedTime()), currentTimestamp
);
domainsById[domainToStore.GetId()] = domainToStore;
ICollection <TimelineDomain> domainsByOneOwner = domainsByOwner[domainToStore.GetOwner
()];
if (domainsByOneOwner == null)
{
domainsByOneOwner = new HashSet <TimelineDomain>();
domainsByOwner[domainToStore.GetOwner()] = domainsByOneOwner;
}
if (domainToReplace != null)
{
domainsByOneOwner.Remove(domainToReplace);
}
domainsByOneOwner.AddItem(domainToStore);
}
/// <summary>Add or update an domain.</summary>
/// <remarks>
/// Add or update an domain. If the domain already exists, only the owner
/// and the admin can update it.
/// </remarks>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual void PutDomain(TimelineDomain domain, UserGroupInformation callerUGI
)
{
TimelineDomain existingDomain = store.GetDomain(domain.GetId());
if (existingDomain != null)
{
if (!timelineACLsManager.CheckAccess(callerUGI, existingDomain))
{
throw new YarnException(callerUGI.GetShortUserName() + " is not allowed to override an existing domain "
+ existingDomain.GetId());
}
// Set it again in case ACLs are not enabled: The domain can be
// modified by every body, but the owner is not changed.
domain.SetOwner(existingDomain.GetOwner());
}
store.Put(domain);
// If the domain exists already, it is likely to be in the cache.
// We need to invalidate it.
if (existingDomain != null)
{
timelineACLsManager.ReplaceIfExist(domain);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public override void PutDomain(TimelineDomain domain)
{
DoPosting(domain, "domain");
}
/// <exception cref="System.Exception"/>
public virtual void TestDSShell(bool haveDomain)
{
string[] args = new string[] { "--jar", AppmasterJar, "--num_containers", "2", "--shell_command"
, Shell.Windows ? "dir" : "ls", "--master_memory", "512", "--master_vcores", "2"
, "--container_memory", "128", "--container_vcores", "1" };
if (haveDomain)
{
string[] domainArgs = new string[] { "--domain", "TEST_DOMAIN", "--view_acls", "reader_user reader_group"
, "--modify_acls", "writer_user writer_group", "--create" };
IList <string> argsList = new AList <string>(Arrays.AsList(args));
Sharpen.Collections.AddAll(argsList, Arrays.AsList(domainArgs));
args = Sharpen.Collections.ToArray(argsList, new string[argsList.Count]);
}
Log.Info("Initializing DS Client");
Client client = new Client(new Configuration(yarnCluster.GetConfig()));
bool initSuccess = client.Init(args);
NUnit.Framework.Assert.IsTrue(initSuccess);
Log.Info("Running DS Client");
AtomicBoolean result = new AtomicBoolean(false);
Sharpen.Thread t = new _Thread_194(result, client);
t.Start();
YarnClient yarnClient = YarnClient.CreateYarnClient();
yarnClient.Init(new Configuration(yarnCluster.GetConfig()));
yarnClient.Start();
string hostName = NetUtils.GetHostname();
bool verified = false;
string errorMessage = string.Empty;
while (!verified)
{
IList <ApplicationReport> apps = yarnClient.GetApplications();
if (apps.Count == 0)
{
Sharpen.Thread.Sleep(10);
continue;
}
ApplicationReport appReport = apps[0];
if (appReport.GetHost().Equals("N/A"))
{
Sharpen.Thread.Sleep(10);
continue;
}
errorMessage = "Expected host name to start with '" + hostName + "', was '" + appReport
.GetHost() + "'. Expected rpc port to be '-1', was '" + appReport.GetRpcPort() +
"'.";
if (CheckHostname(appReport.GetHost()) && appReport.GetRpcPort() == -1)
{
verified = true;
}
if (appReport.GetYarnApplicationState() == YarnApplicationState.Finished)
{
break;
}
}
NUnit.Framework.Assert.IsTrue(errorMessage, verified);
t.Join();
Log.Info("Client run completed. Result=" + result);
NUnit.Framework.Assert.IsTrue(result.Get());
TimelineDomain domain = null;
if (haveDomain)
{
domain = yarnCluster.GetApplicationHistoryServer().GetTimelineStore().GetDomain("TEST_DOMAIN"
);
NUnit.Framework.Assert.IsNotNull(domain);
NUnit.Framework.Assert.AreEqual("reader_user reader_group", domain.GetReaders());
NUnit.Framework.Assert.AreEqual("writer_user writer_group", domain.GetWriters());
}
TimelineEntities entitiesAttempts = yarnCluster.GetApplicationHistoryServer().GetTimelineStore
().GetEntities(ApplicationMaster.DSEntity.DsAppAttempt.ToString(), null, null, null
, null, null, null, null, null, null);
NUnit.Framework.Assert.IsNotNull(entitiesAttempts);
NUnit.Framework.Assert.AreEqual(1, entitiesAttempts.GetEntities().Count);
NUnit.Framework.Assert.AreEqual(2, entitiesAttempts.GetEntities()[0].GetEvents().
Count);
NUnit.Framework.Assert.AreEqual(entitiesAttempts.GetEntities()[0].GetEntityType()
.ToString(), ApplicationMaster.DSEntity.DsAppAttempt.ToString());
if (haveDomain)
{
NUnit.Framework.Assert.AreEqual(domain.GetId(), entitiesAttempts.GetEntities()[0]
.GetDomainId());
}
else
{
NUnit.Framework.Assert.AreEqual("DEFAULT", entitiesAttempts.GetEntities()[0].GetDomainId
());
}
TimelineEntities entities = yarnCluster.GetApplicationHistoryServer().GetTimelineStore
().GetEntities(ApplicationMaster.DSEntity.DsContainer.ToString(), null, null, null
, null, null, null, null, null, null);
NUnit.Framework.Assert.IsNotNull(entities);
NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count);
NUnit.Framework.Assert.AreEqual(entities.GetEntities()[0].GetEntityType().ToString
(), ApplicationMaster.DSEntity.DsContainer.ToString());
if (haveDomain)
{
NUnit.Framework.Assert.AreEqual(domain.GetId(), entities.GetEntities()[0].GetDomainId
());
}
else
{
NUnit.Framework.Assert.AreEqual("DEFAULT", entities.GetEntities()[0].GetDomainId(
));
}
}
public abstract void PutDomain(TimelineDomain domain);
