/// <exception cref="System.IO.IOException"/>
public override void StoreToken(TimelineDelegationTokenIdentifier tokenId, long renewDate
)
{
DataOutputStream ds = null;
WriteBatch batch = null;
try
{
byte[] k = CreateTokenEntryKey(tokenId.GetSequenceNumber());
if (db.Get(k) != null)
{
throw new IOException(tokenId + " already exists");
}
byte[] v = BuildTokenData(tokenId, renewDate);
ByteArrayOutputStream bs = new ByteArrayOutputStream();
ds = new DataOutputStream(bs);
ds.WriteInt(tokenId.GetSequenceNumber());
batch = db.CreateWriteBatch();
batch.Put(k, v);
batch.Put(LatestSequenceNumberKey, bs.ToByteArray());
db.Write(batch);
}
catch (DBException e)
{
throw new IOException(e);
}
finally
{
IOUtils.Cleanup(Log, ds);
IOUtils.Cleanup(Log, batch);
}
}
/// <exception cref="System.IO.IOException"/>
private static byte[] BuildTokenData(TimelineDelegationTokenIdentifier tokenId, long
renewDate)
{
TimelineDelegationTokenIdentifierData data = new TimelineDelegationTokenIdentifierData
(tokenId, renewDate);
return(data.ToByteArray());
}
/// <exception cref="System.IO.IOException"/>
public override void UpdateToken(TimelineDelegationTokenIdentifier tokenId, long
renewDate)
{
if (!state.tokenState.Contains(tokenId))
{
throw new IOException("token " + tokenId + " not in store");
}
state.tokenState[tokenId] = renewDate;
}
/// <exception cref="System.IO.IOException"/>
public virtual TimelineDelegationTokenIdentifier GetTokenIdentifier()
{
ByteArrayInputStream @in = new ByteArrayInputStream(builder.GetTokenIdentifier().
ToByteArray());
TimelineDelegationTokenIdentifier identifer = new TimelineDelegationTokenIdentifier
();
identifer.ReadFields(new DataInputStream(@in));
return(identifer);
}
/// <exception cref="System.IO.IOException"/>
public override void StoreToken(TimelineDelegationTokenIdentifier tokenId, long renewDate
)
{
if (state.tokenState.Contains(tokenId))
{
throw new IOException("token " + tokenId + " was stored twice");
}
state.tokenState[tokenId] = renewDate;
state.latestSequenceNumber = tokenId.GetSequenceNumber();
}
/// <exception cref="System.IO.IOException"/>
public override void RemoveToken(TimelineDelegationTokenIdentifier tokenId)
{
try
{
byte[] key = CreateTokenEntryKey(tokenId.GetSequenceNumber());
db.Delete(key);
}
catch (DBException e)
{
throw new IOException(e);
}
}
public virtual void TestParseTimelineDelegationTokenIdentifierRenewer()
{
// Server side when generation a timeline DT
Configuration conf = new YarnConfiguration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthToLocal, "RULE:[2:$1@$0]([nr]m@.*EXAMPLE.COM)s/.*/yarn/"
);
HadoopKerberosName.SetConfiguration(conf);
Text owner = new Text("owner");
Text renewer = new Text("rm/[email protected]");
Text realUser = new Text("realUser");
TimelineDelegationTokenIdentifier token = new TimelineDelegationTokenIdentifier(owner
, renewer, realUser);
NUnit.Framework.Assert.AreEqual(new Text("yarn"), token.GetRenewer());
}
/// <exception cref="System.IO.IOException"/>
public override void UpdateToken(TimelineDelegationTokenIdentifier tokenId, long
renewDate)
{
try
{
byte[] k = CreateTokenEntryKey(tokenId.GetSequenceNumber());
if (db.Get(k) == null)
{
throw new IOException(tokenId + " doesn't exist");
}
byte[] v = BuildTokenData(tokenId, renewDate);
db.Put(k, v);
}
catch (DBException e)
{
throw new IOException(e);
}
}
public virtual void TestTimelineDelegationTokenIdentifier()
{
Text owner = new Text("user1");
Text renewer = new Text("user2");
Text realUser = new Text("user3");
long issueDate = 1;
long maxDate = 2;
int sequenceNumber = 3;
int masterKeyId = 4;
TimelineDelegationTokenIdentifier token = new TimelineDelegationTokenIdentifier(owner
, renewer, realUser);
token.SetIssueDate(issueDate);
token.SetMaxDate(maxDate);
token.SetSequenceNumber(sequenceNumber);
token.SetMasterKeyId(masterKeyId);
TimelineDelegationTokenIdentifier anotherToken = new TimelineDelegationTokenIdentifier
();
byte[] tokenContent = token.GetBytes();
DataInputBuffer dib = new DataInputBuffer();
dib.Reset(tokenContent, tokenContent.Length);
anotherToken.ReadFields(dib);
// verify the whole record equals with original record
NUnit.Framework.Assert.AreEqual("Token is not the same after serialization " + "and deserialization."
, token, anotherToken);
NUnit.Framework.Assert.AreEqual("owner from proto is not the same with original token"
, anotherToken.GetOwner(), owner);
NUnit.Framework.Assert.AreEqual("renewer from proto is not the same with original token"
, anotherToken.GetRenewer(), renewer);
NUnit.Framework.Assert.AreEqual("realUser from proto is not the same with original token"
, anotherToken.GetRealUser(), realUser);
NUnit.Framework.Assert.AreEqual("issueDate from proto is not the same with original token"
, anotherToken.GetIssueDate(), issueDate);
NUnit.Framework.Assert.AreEqual("maxDate from proto is not the same with original token"
, anotherToken.GetMaxDate(), maxDate);
NUnit.Framework.Assert.AreEqual("sequenceNumber from proto is not the same with original token"
, anotherToken.GetSequenceNumber(), sequenceNumber);
NUnit.Framework.Assert.AreEqual("masterKeyId from proto is not the same with original token"
, anotherToken.GetMasterKeyId(), masterKeyId);
}
public TimelineDelegationTokenIdentifierData(TimelineDelegationTokenIdentifier identifier
, long renewdate)
{
builder.SetTokenIdentifier(identifier.GetProto());
builder.SetRenewDate(renewdate);
}
public virtual void TestDelegationTokenOperationsRetry()
{
int newMaxRetries = 5;
long newIntervalMs = 500;
YarnConfiguration conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.TimelineServiceClientMaxRetries, newMaxRetries);
conf.SetLong(YarnConfiguration.TimelineServiceClientRetryIntervalMs, newIntervalMs
);
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
// use kerberos to bypass the issue in HADOOP-11215
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
TimelineClientImpl client = CreateTimelineClient(conf);
TestTimelineClient.TestTimlineDelegationTokenSecretManager dtManager = new TestTimelineClient.TestTimlineDelegationTokenSecretManager
();
try
{
dtManager.StartThreads();
Sharpen.Thread.Sleep(3000);
try
{
// try getting a delegation token
client.GetDelegationToken(UserGroupInformation.GetCurrentUser().GetShortUserName(
));
AssertFail();
}
catch (RuntimeException ce)
{
AssertException(client, ce);
}
try
{
// try renew a delegation token
TimelineDelegationTokenIdentifier timelineDT = new TimelineDelegationTokenIdentifier
(new Text("tester"), new Text("tester"), new Text("tester"));
client.RenewDelegationToken(new Org.Apache.Hadoop.Security.Token.Token <TimelineDelegationTokenIdentifier
>(timelineDT.GetBytes(), dtManager.CreatePassword(timelineDT), timelineDT.GetKind
(), new Text("0.0.0.0:8188")));
AssertFail();
}
catch (RuntimeException ce)
{
AssertException(client, ce);
}
try
{
// try cancel a delegation token
TimelineDelegationTokenIdentifier timelineDT = new TimelineDelegationTokenIdentifier
(new Text("tester"), new Text("tester"), new Text("tester"));
client.CancelDelegationToken(new Org.Apache.Hadoop.Security.Token.Token <TimelineDelegationTokenIdentifier
>(timelineDT.GetBytes(), dtManager.CreatePassword(timelineDT), timelineDT.GetKind
(), new Text("0.0.0.0:8188")));
AssertFail();
}
catch (RuntimeException ce)
{
AssertException(client, ce);
}
}
finally
{
client.Stop();
dtManager.StopThreads();
}
}
public virtual void TestTokenStore()
{
InitAndStartTimelineServiceStateStoreService();
TimelineStateStore.TimelineServiceState state = store.LoadState();
NUnit.Framework.Assert.IsTrue("token state not empty", state.tokenState.IsEmpty()
);
NUnit.Framework.Assert.IsTrue("key state not empty", state.tokenMasterKeyState.IsEmpty
());
DelegationKey key1 = new DelegationKey(1, 2, Sharpen.Runtime.GetBytesForString("keyData1"
));
TimelineDelegationTokenIdentifier token1 = new TimelineDelegationTokenIdentifier(
new Text("tokenOwner1"), new Text("tokenRenewer1"), new Text("tokenUser1"));
token1.SetSequenceNumber(1);
token1.GetBytes();
long tokenDate1 = 1L;
TimelineDelegationTokenIdentifier token2 = new TimelineDelegationTokenIdentifier(
new Text("tokenOwner2"), new Text("tokenRenewer2"), new Text("tokenUser2"));
token2.SetSequenceNumber(12345678);
token2.GetBytes();
long tokenDate2 = 87654321L;
store.StoreTokenMasterKey(key1);
try
{
store.StoreTokenMasterKey(key1);
NUnit.Framework.Assert.Fail("redundant store of key undetected");
}
catch (IOException)
{
}
// expected
store.StoreToken(token1, tokenDate1);
store.StoreToken(token2, tokenDate2);
try
{
store.StoreToken(token1, tokenDate1);
NUnit.Framework.Assert.Fail("redundant store of token undetected");
}
catch (IOException)
{
}
// expected
store.Close();
InitAndStartTimelineServiceStateStoreService();
state = store.LoadState();
NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState
.Count);
NUnit.Framework.Assert.IsTrue("missing token 1", state.tokenState.Contains(token1
));
NUnit.Framework.Assert.AreEqual("incorrect token 1 date", tokenDate1, state.tokenState
[token1]);
NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2
));
NUnit.Framework.Assert.AreEqual("incorrect token 2 date", tokenDate2, state.tokenState
[token2]);
NUnit.Framework.Assert.AreEqual("incorrect master key count", 1, state.tokenMasterKeyState
.Count);
NUnit.Framework.Assert.IsTrue("missing master key 1", state.tokenMasterKeyState.Contains
(key1));
NUnit.Framework.Assert.AreEqual("incorrect latest sequence number", 12345678, state
.GetLatestSequenceNumber());
DelegationKey key2 = new DelegationKey(3, 4, Sharpen.Runtime.GetBytesForString("keyData2"
));
DelegationKey key3 = new DelegationKey(5, 6, Sharpen.Runtime.GetBytesForString("keyData3"
));
TimelineDelegationTokenIdentifier token3 = new TimelineDelegationTokenIdentifier(
new Text("tokenOwner3"), new Text("tokenRenewer3"), new Text("tokenUser3"));
token3.SetSequenceNumber(12345679);
token3.GetBytes();
long tokenDate3 = 87654321L;
store.RemoveToken(token1);
store.StoreTokenMasterKey(key2);
long newTokenDate2 = 975318642L;
store.UpdateToken(token2, newTokenDate2);
store.RemoveTokenMasterKey(key1);
store.StoreTokenMasterKey(key3);
store.StoreToken(token3, tokenDate3);
store.Close();
InitAndStartTimelineServiceStateStoreService();
state = store.LoadState();
NUnit.Framework.Assert.AreEqual("incorrect loaded token count", 2, state.tokenState
.Count);
NUnit.Framework.Assert.IsFalse("token 1 not removed", state.tokenState.Contains(token1
));
NUnit.Framework.Assert.IsTrue("missing token 2", state.tokenState.Contains(token2
));
NUnit.Framework.Assert.AreEqual("incorrect token 2 date", newTokenDate2, state.tokenState
[token2]);
NUnit.Framework.Assert.IsTrue("missing token 3", state.tokenState.Contains(token3
));
NUnit.Framework.Assert.AreEqual("incorrect token 3 date", tokenDate3, state.tokenState
[token3]);
NUnit.Framework.Assert.AreEqual("incorrect master key count", 2, state.tokenMasterKeyState
.Count);
NUnit.Framework.Assert.IsFalse("master key 1 not removed", state.tokenMasterKeyState
.Contains(key1));
NUnit.Framework.Assert.IsTrue("missing master key 2", state.tokenMasterKeyState.Contains
(key2));
NUnit.Framework.Assert.IsTrue("missing master key 3", state.tokenMasterKeyState.Contains
(key3));
NUnit.Framework.Assert.AreEqual("incorrect latest sequence number", 12345679, state
.GetLatestSequenceNumber());
store.Close();
}
/// <summary>Blocking method to remove a delegation token from the state storage.</summary> /// <remarks> /// Blocking method to remove a delegation token from the state storage. /// Implementations must not return from this method until the token has been /// removed from the state store. /// </remarks> /// <param name="tokenId">the token to remove</param> /// <exception cref="System.IO.IOException"/> public abstract void RemoveToken(TimelineDelegationTokenIdentifier tokenId);
/// <summary>
/// Blocking method to update the expiration of a delegation token
/// in the state storage.
/// </summary>
/// <remarks>
/// Blocking method to update the expiration of a delegation token
/// in the state storage.
/// Implementations must not return from this method until the expiration
/// date of the token has been updated in the state store.
/// </remarks>
/// <param name="tokenId">the token to update</param>
/// <param name="renewDate">the new token renewal deadline</param>
/// <exception cref="System.IO.IOException"/>
public abstract void UpdateToken(TimelineDelegationTokenIdentifier tokenId, long
renewDate);
public virtual void TestDelegationTokenOperations()
{
TimelineClient httpUserClient = KerberosTestUtils.DoAs(HttpUser + "/localhost", new
_Callable_221(this));
UserGroupInformation httpUser = KerberosTestUtils.DoAs(HttpUser + "/localhost", new
_Callable_228());
// Let HTTP user to get the delegation for itself
Org.Apache.Hadoop.Security.Token.Token <TimelineDelegationTokenIdentifier> token =
httpUserClient.GetDelegationToken(httpUser.GetShortUserName());
NUnit.Framework.Assert.IsNotNull(token);
TimelineDelegationTokenIdentifier tDT = token.DecodeIdentifier();
NUnit.Framework.Assert.IsNotNull(tDT);
NUnit.Framework.Assert.AreEqual(new Text(HttpUser), tDT.GetOwner());
// Renew token
NUnit.Framework.Assert.IsFalse(token.GetService().ToString().IsEmpty());
// Renew the token from the token service address
long renewTime1 = httpUserClient.RenewDelegationToken(token);
Sharpen.Thread.Sleep(100);
token.SetService(new Text());
NUnit.Framework.Assert.IsTrue(token.GetService().ToString().IsEmpty());
// If the token service address is not avaiable, it still can be renewed
// from the configured address
long renewTime2 = httpUserClient.RenewDelegationToken(token);
NUnit.Framework.Assert.IsTrue(renewTime1 < renewTime2);
// Cancel token
NUnit.Framework.Assert.IsTrue(token.GetService().ToString().IsEmpty());
// If the token service address is not avaiable, it still can be canceled
// from the configured address
httpUserClient.CancelDelegationToken(token);
// Renew should not be successful because the token is canceled
try
{
httpUserClient.RenewDelegationToken(token);
NUnit.Framework.Assert.Fail();
}
catch (Exception e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("Renewal request for unknown token"
));
}
// Let HTTP user to get the delegation token for FOO user
UserGroupInformation fooUgi = UserGroupInformation.CreateProxyUser(FooUser, httpUser
);
TimelineClient fooUserClient = fooUgi.DoAs(new _PrivilegedExceptionAction_272(this
));
token = fooUserClient.GetDelegationToken(httpUser.GetShortUserName());
NUnit.Framework.Assert.IsNotNull(token);
tDT = token.DecodeIdentifier();
NUnit.Framework.Assert.IsNotNull(tDT);
NUnit.Framework.Assert.AreEqual(new Text(FooUser), tDT.GetOwner());
NUnit.Framework.Assert.AreEqual(new Text(HttpUser), tDT.GetRealUser());
// Renew token as the renewer
Org.Apache.Hadoop.Security.Token.Token <TimelineDelegationTokenIdentifier> tokenToRenew
= token;
renewTime1 = httpUserClient.RenewDelegationToken(tokenToRenew);
renewTime2 = httpUserClient.RenewDelegationToken(tokenToRenew);
NUnit.Framework.Assert.IsTrue(renewTime1 < renewTime2);
// Cancel token
NUnit.Framework.Assert.IsFalse(tokenToRenew.GetService().ToString().IsEmpty());
// Cancel the token from the token service address
fooUserClient.CancelDelegationToken(tokenToRenew);
// Renew should not be successful because the token is canceled
try
{
httpUserClient.RenewDelegationToken(tokenToRenew);
NUnit.Framework.Assert.Fail();
}
catch (Exception e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("Renewal request for unknown token"
));
}
// Let HTTP user to get the delegation token for BAR user
UserGroupInformation barUgi = UserGroupInformation.CreateProxyUser(BarUser, httpUser
);
TimelineClient barUserClient = barUgi.DoAs(new _PrivilegedExceptionAction_309(this
));
try
{
barUserClient.GetDelegationToken(httpUser.GetShortUserName());
NUnit.Framework.Assert.Fail();
}
catch (Exception e)
{
NUnit.Framework.Assert.IsTrue(e.InnerException is AuthorizationException || e.InnerException
is AuthenticationException);
}
}
/// <exception cref="System.IO.IOException"/>
public override void RemoveToken(TimelineDelegationTokenIdentifier tokenId)
{
Sharpen.Collections.Remove(state.tokenState, tokenId);
}
