/// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/>
internal static void CheckPermissionForApi(FSPermissionChecker pc, XAttr xAttr, bool
isRawPath)
{
bool isSuperUser = pc.IsSuperUser();
if (xAttr.GetNameSpace() == XAttr.NameSpace.User || (xAttr.GetNameSpace() == XAttr.NameSpace
.Trusted && isSuperUser))
{
return;
}
if (xAttr.GetNameSpace() == XAttr.NameSpace.Raw && isRawPath && isSuperUser)
{
return;
}
if (XAttrHelper.GetPrefixName(xAttr).Equals(HdfsServerConstants.SecurityXattrUnreadableBySuperuser
))
{
if (xAttr.GetValue() != null)
{
throw new AccessControlException("Attempt to set a value for '" + HdfsServerConstants
.SecurityXattrUnreadableBySuperuser + "'. Values are not allowed for this xattr."
);
}
return;
}
throw new AccessControlException("User doesn't have permission for xattr: " + XAttrHelper
.GetPrefixName(xAttr));
}
/// <summary>Create a new encryption zone.</summary>
/// <remarks>
/// Create a new encryption zone.
/// <p/>
/// Called while holding the FSDirectory lock.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
internal virtual XAttr CreateEncryptionZone(string src, CipherSuite suite, CryptoProtocolVersion
version, string keyName)
{
System.Diagnostics.Debug.Assert(dir.HasWriteLock());
INodesInPath srcIIP = dir.GetINodesInPath4Write(src, false);
if (dir.IsNonEmptyDirectory(srcIIP))
{
throw new IOException("Attempt to create an encryption zone for a non-empty directory."
);
}
if (srcIIP != null && srcIIP.GetLastINode() != null && !srcIIP.GetLastINode().IsDirectory
())
{
throw new IOException("Attempt to create an encryption zone for a file.");
}
EncryptionZoneManager.EncryptionZoneInt ezi = GetEncryptionZoneForPath(srcIIP);
if (ezi != null)
{
throw new IOException("Directory " + src + " is already in an " + "encryption zone. ("
+ GetFullPathName(ezi) + ")");
}
HdfsProtos.ZoneEncryptionInfoProto proto = PBHelper.Convert(suite, version, keyName
);
XAttr ezXAttr = XAttrHelper.BuildXAttr(HdfsServerConstants.CryptoXattrEncryptionZone
, proto.ToByteArray());
IList <XAttr> xattrs = Lists.NewArrayListWithCapacity(1);
xattrs.AddItem(ezXAttr);
// updating the xattr will call addEncryptionZone,
// done this way to handle edit log loading
FSDirXAttrOp.UnprotectedSetXAttrs(dir, src, xattrs, EnumSet.Of(XAttrSetFlag.Create
));
return(ezXAttr);
}
/// <summary>Set xattr for a file or directory.</summary>
/// <param name="src">- path on which it sets the xattr</param>
/// <param name="xAttr">- xAttr details to set</param>
/// <param name="flag">- xAttrs flags</param>
/// <exception cref="System.IO.IOException"/>
internal static HdfsFileStatus SetXAttr(FSDirectory fsd, string src, XAttr xAttr,
EnumSet <XAttrSetFlag> flag, bool logRetryCache)
{
CheckXAttrsConfigFlag(fsd);
CheckXAttrSize(fsd, xAttr);
FSPermissionChecker pc = fsd.GetPermissionChecker();
XAttrPermissionFilter.CheckPermissionForApi(pc, xAttr, FSDirectory.IsReservedRawName
(src));
byte[][] pathComponents = FSDirectory.GetPathComponentsForReservedPath(src);
src = fsd.ResolvePath(pc, src, pathComponents);
IList <XAttr> xAttrs = Lists.NewArrayListWithCapacity(1);
xAttrs.AddItem(xAttr);
INodesInPath iip;
fsd.WriteLock();
try
{
iip = fsd.GetINodesInPath4Write(src);
CheckXAttrChangeAccess(fsd, iip, xAttr, pc);
UnprotectedSetXAttrs(fsd, src, xAttrs, flag);
}
finally
{
fsd.WriteUnlock();
}
fsd.GetEditLog().LogSetXAttrs(src, xAttrs, logRetryCache);
return(fsd.GetAuditFileInfo(iip));
}
public virtual void TestToXAttrMap()
{
string jsonString = "{\"XAttrs\":[{\"name\":\"user.a1\",\"value\":\"0x313233\"},"
+ "{\"name\":\"user.a2\",\"value\":\"0x313131\"}]}";
ObjectReader reader = new ObjectMapper().Reader(typeof(IDictionary));
IDictionary <object, object> json = reader.ReadValue(jsonString);
XAttr xAttr1 = (new XAttr.Builder()).SetNameSpace(XAttr.NameSpace.User).SetName("a1"
).SetValue(XAttrCodec.DecodeValue("0x313233")).Build();
XAttr xAttr2 = (new XAttr.Builder()).SetNameSpace(XAttr.NameSpace.User).SetName("a2"
).SetValue(XAttrCodec.DecodeValue("0x313131")).Build();
IList <XAttr> xAttrs = Lists.NewArrayList();
xAttrs.AddItem(xAttr1);
xAttrs.AddItem(xAttr2);
IDictionary <string, byte[]> xAttrMap = XAttrHelper.BuildXAttrMap(xAttrs);
IDictionary <string, byte[]> parsedXAttrMap = JsonUtil.ToXAttrs(json);
NUnit.Framework.Assert.AreEqual(xAttrMap.Count, parsedXAttrMap.Count);
IEnumerator <KeyValuePair <string, byte[]> > iter = xAttrMap.GetEnumerator();
while (iter.HasNext())
{
KeyValuePair <string, byte[]> entry = iter.Next();
Assert.AssertArrayEquals(entry.Value, parsedXAttrMap[entry.Key]);
}
}
/// <summary>Build xattr name with prefix as <code>XAttr</code> list.</summary>
public static IList <XAttr> BuildXAttrAsList(string name)
{
XAttr xAttr = BuildXAttr(name);
IList <XAttr> xAttrs = Lists.NewArrayListWithCapacity(1);
xAttrs.AddItem(xAttr);
return(xAttrs);
}
/// <summary>Get name with prefix from <code>XAttr</code></summary>
public static string GetPrefixName(XAttr xAttr)
{
if (xAttr == null)
{
return(null);
}
string @namespace = xAttr.GetNameSpace().ToString();
return(StringUtils.ToLowerCase(@namespace) + "." + xAttr.GetName());
}
/// <exception cref="System.IO.IOException"/>
private static void SetDirStoragePolicy(FSDirectory fsd, INodeDirectory inode, byte
policyId, int latestSnapshotId)
{
IList <XAttr> existingXAttrs = XAttrStorage.ReadINodeXAttrs(inode);
XAttr xAttr = BlockStoragePolicySuite.BuildXAttr(policyId);
IList <XAttr> newXAttrs = FSDirXAttrOp.SetINodeXAttrs(fsd, existingXAttrs, Arrays.
AsList(xAttr), EnumSet.Of(XAttrSetFlag.Create, XAttrSetFlag.Replace));
XAttrStorage.UpdateINodeXAttrs(inode, newXAttrs, latestSnapshotId);
}
/// <summary>
/// Verify that the first <i>num</i> generatedXAttrs are present in
/// newXAttrs.
/// </summary>
private static void VerifyXAttrsPresent(IList <XAttr> newXAttrs, int num)
{
NUnit.Framework.Assert.AreEqual("Unexpected number of XAttrs after multiset", num
, newXAttrs.Count);
for (int i = 0; i < num; i++)
{
XAttr search = generatedXAttrs[i];
NUnit.Framework.Assert.IsTrue("Did not find set XAttr " + search + " + after multiset"
, newXAttrs.Contains(search));
}
}
private static IList <XAttr> GenerateXAttrs(int numXAttrs)
{
IList <XAttr> generatedXAttrs = Lists.NewArrayListWithCapacity(numXAttrs);
for (int i = 0; i < numXAttrs; i++)
{
XAttr xAttr = (new XAttr.Builder()).SetNameSpace(XAttr.NameSpace.System).SetName(
"a" + i).SetValue(new byte[] { unchecked ((byte)i), unchecked ((byte)(i + 1)), unchecked (
(byte)(i + 2)) }).Build();
generatedXAttrs.AddItem(xAttr);
}
return(generatedXAttrs);
}
public virtual void TestToJsonFromXAttrs()
{
string jsonString = "{\"XAttrs\":[{\"name\":\"user.a1\",\"value\":\"0x313233\"},"
+ "{\"name\":\"user.a2\",\"value\":\"0x313131\"}]}";
XAttr xAttr1 = (new XAttr.Builder()).SetNameSpace(XAttr.NameSpace.User).SetName("a1"
).SetValue(XAttrCodec.DecodeValue("0x313233")).Build();
XAttr xAttr2 = (new XAttr.Builder()).SetNameSpace(XAttr.NameSpace.User).SetName("a2"
).SetValue(XAttrCodec.DecodeValue("0x313131")).Build();
IList <XAttr> xAttrs = Lists.NewArrayList();
xAttrs.AddItem(xAttr1);
xAttrs.AddItem(xAttr2);
NUnit.Framework.Assert.AreEqual(jsonString, JsonUtil.ToJsonString(xAttrs, XAttrCodec
.Hex));
}
/// <exception cref="System.IO.IOException"/>
private static IDictionary <string, object> ToJsonMap(XAttr xAttr, XAttrCodec encoding
)
{
if (xAttr == null)
{
return(null);
}
IDictionary <string, object> m = new SortedDictionary <string, object>();
m["name"] = XAttrHelper.GetPrefixName(xAttr);
m["value"] = xAttr.GetValue() != null?XAttrCodec.EncodeValue(xAttr.GetValue(),
encoding) : null;
return(m);
}
/// <summary>Get value of first xattr from <code>XAttr</code> list</summary>
public static byte[] GetFirstXAttrValue(IList <XAttr> xAttrs)
{
byte[] value = null;
XAttr xAttr = GetFirstXAttr(xAttrs);
if (xAttr != null)
{
value = xAttr.GetValue();
if (value == null)
{
value = new byte[0];
}
}
// xattr exists, but no value.
return(value);
}
public virtual void TestINodeXAttrsLimit()
{
IList <XAttr> existingXAttrs = Lists.NewArrayListWithCapacity(2);
XAttr xAttr1 = (new XAttr.Builder()).SetNameSpace(XAttr.NameSpace.User).SetName("a1"
).SetValue(new byte[] { unchecked ((int)(0x31)), unchecked ((int)(0x32)), unchecked (
(int)(0x33)) }).Build();
XAttr xAttr2 = (new XAttr.Builder()).SetNameSpace(XAttr.NameSpace.User).SetName("a2"
).SetValue(new byte[] { unchecked ((int)(0x31)), unchecked ((int)(0x31)), unchecked (
(int)(0x31)) }).Build();
existingXAttrs.AddItem(xAttr1);
existingXAttrs.AddItem(xAttr2);
// Adding system and raw namespace xAttrs aren't affected by inode
// xAttrs limit.
XAttr newSystemXAttr = (new XAttr.Builder()).SetNameSpace(XAttr.NameSpace.System)
.SetName("a3").SetValue(new byte[] { unchecked ((int)(0x33)), unchecked ((int)(0x33
)), unchecked ((int)(0x33)) }).Build();
XAttr newRawXAttr = (new XAttr.Builder()).SetNameSpace(XAttr.NameSpace.Raw).SetName
("a3").SetValue(new byte[] { unchecked ((int)(0x33)), unchecked ((int)(0x33)), unchecked (
(int)(0x33)) }).Build();
IList <XAttr> newXAttrs = Lists.NewArrayListWithCapacity(2);
newXAttrs.AddItem(newSystemXAttr);
newXAttrs.AddItem(newRawXAttr);
IList <XAttr> xAttrs = FSDirXAttrOp.SetINodeXAttrs(fsdir, existingXAttrs, newXAttrs
, EnumSet.Of(XAttrSetFlag.Create, XAttrSetFlag.Replace));
NUnit.Framework.Assert.AreEqual(xAttrs.Count, 4);
// Adding a trusted namespace xAttr, is affected by inode xAttrs limit.
XAttr newXAttr1 = (new XAttr.Builder()).SetNameSpace(XAttr.NameSpace.Trusted).SetName
("a4").SetValue(new byte[] { unchecked ((int)(0x34)), unchecked ((int)(0x34)), unchecked (
(int)(0x34)) }).Build();
newXAttrs.Set(0, newXAttr1);
try
{
FSDirXAttrOp.SetINodeXAttrs(fsdir, existingXAttrs, newXAttrs, EnumSet.Of(XAttrSetFlag
.Create, XAttrSetFlag.Replace));
NUnit.Framework.Assert.Fail("Setting user visible xattr on inode should fail if "
+ "reaching limit.");
}
catch (IOException e)
{
GenericTestUtils.AssertExceptionContains("Cannot add additional XAttr " + "to inode, would exceed limit"
, e);
}
}
/// <summary>
/// Verifies that the combined size of the name and value of an xattr is within
/// the configured limit.
/// </summary>
/// <remarks>
/// Verifies that the combined size of the name and value of an xattr is within
/// the configured limit. Setting a limit of zero disables this check.
/// </remarks>
private static void CheckXAttrSize(FSDirectory fsd, XAttr xAttr)
{
if (fsd.GetXattrMaxSize() == 0)
{
return;
}
int size = Sharpen.Runtime.GetBytesForString(xAttr.GetName(), Charsets.Utf8).Length;
if (xAttr.GetValue() != null)
{
size += xAttr.GetValue().Length;
}
if (size > fsd.GetXattrMaxSize())
{
throw new HadoopIllegalArgumentException("The XAttr is too big. The maximum combined size of the"
+ " name and value is " + fsd.GetXattrMaxSize() + ", but the total size is " +
size);
}
}
internal static IList <XAttr> FilterINodeXAttrs(IList <XAttr> existingXAttrs, IList
<XAttr> toFilter, IList <XAttr> filtered)
{
if (existingXAttrs == null || existingXAttrs.IsEmpty() || toFilter == null || toFilter
.IsEmpty())
{
return(existingXAttrs);
}
// Populate a new list with XAttrs that pass the filter
IList <XAttr> newXAttrs = Lists.NewArrayListWithCapacity(existingXAttrs.Count);
foreach (XAttr a in existingXAttrs)
{
bool add = true;
for (ListIterator <XAttr> it = toFilter.ListIterator(); it.HasNext();)
{
XAttr filter = it.Next();
Preconditions.CheckArgument(!KeyidXattr.EqualsIgnoreValue(filter), "The encryption zone xattr should never be deleted."
);
if (UnreadableBySuperuserXattr.EqualsIgnoreValue(filter))
{
throw new AccessControlException("The xattr '" + HdfsServerConstants.SecurityXattrUnreadableBySuperuser
+ "' can not be deleted.");
}
if (a.EqualsIgnoreValue(filter))
{
add = false;
it.Remove();
filtered.AddItem(filter);
break;
}
}
if (add)
{
newXAttrs.AddItem(a);
}
}
return(newXAttrs);
}
/// <summary>Remove an xattr for a file or directory.</summary>
/// <param name="src">- path to remove the xattr from</param>
/// <param name="xAttr">- xAttr to remove</param>
/// <exception cref="System.IO.IOException"/>
internal static HdfsFileStatus RemoveXAttr(FSDirectory fsd, string src, XAttr xAttr
, bool logRetryCache)
{
FSDirXAttrOp.CheckXAttrsConfigFlag(fsd);
FSPermissionChecker pc = fsd.GetPermissionChecker();
XAttrPermissionFilter.CheckPermissionForApi(pc, xAttr, FSDirectory.IsReservedRawName
(src));
byte[][] pathComponents = FSDirectory.GetPathComponentsForReservedPath(src);
IList <XAttr> xAttrs = Lists.NewArrayListWithCapacity(1);
xAttrs.AddItem(xAttr);
INodesInPath iip;
fsd.WriteLock();
try
{
src = fsd.ResolvePath(pc, src, pathComponents);
iip = fsd.GetINodesInPath4Write(src);
CheckXAttrChangeAccess(fsd, iip, xAttr, pc);
IList <XAttr> removedXAttrs = UnprotectedRemoveXAttrs(fsd, src, xAttrs);
if (removedXAttrs != null && !removedXAttrs.IsEmpty())
{
fsd.GetEditLog().LogRemoveXAttrs(src, removedXAttrs, logRetryCache);
}
else
{
throw new IOException("No matching attributes found for remove operation");
}
}
finally
{
fsd.WriteUnlock();
}
return(fsd.GetAuditFileInfo(iip));
}
/// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/>
private static void CheckXAttrChangeAccess(FSDirectory fsd, INodesInPath iip, XAttr
xAttr, FSPermissionChecker pc)
{
if (fsd.IsPermissionEnabled() && xAttr.GetNameSpace() == XAttr.NameSpace.User)
{
INode inode = iip.GetLastINode();
if (inode != null && inode.IsDirectory() && inode.GetFsPermission().GetStickyBit(
))
{
if (!pc.IsSuperUser())
{
fsd.CheckOwner(pc, iip);
}
}
else
{
fsd.CheckPathAccess(pc, iip, FsAction.Write);
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestXAttrMultiAddRemoveErrors()
{
// Test that the same XAttr can not be multiset twice
IList <XAttr> existingXAttrs = Lists.NewArrayList();
IList <XAttr> toAdd = Lists.NewArrayList();
toAdd.AddItem(generatedXAttrs[0]);
toAdd.AddItem(generatedXAttrs[1]);
toAdd.AddItem(generatedXAttrs[2]);
toAdd.AddItem(generatedXAttrs[0]);
try
{
FSDirXAttrOp.SetINodeXAttrs(fsdir, existingXAttrs, toAdd, EnumSet.Of(XAttrSetFlag
.Create));
NUnit.Framework.Assert.Fail("Specified the same xattr to be set twice");
}
catch (IOException e)
{
GenericTestUtils.AssertExceptionContains("Cannot specify the same " + "XAttr to be set"
, e);
}
// Test that CREATE and REPLACE flags are obeyed
toAdd.Remove(generatedXAttrs[0]);
existingXAttrs.AddItem(generatedXAttrs[0]);
try
{
FSDirXAttrOp.SetINodeXAttrs(fsdir, existingXAttrs, toAdd, EnumSet.Of(XAttrSetFlag
.Create));
NUnit.Framework.Assert.Fail("Set XAttr that is already set without REPLACE flag");
}
catch (IOException e)
{
GenericTestUtils.AssertExceptionContains("already exists", e);
}
try
{
FSDirXAttrOp.SetINodeXAttrs(fsdir, existingXAttrs, toAdd, EnumSet.Of(XAttrSetFlag
.Replace));
NUnit.Framework.Assert.Fail("Set XAttr that does not exist without the CREATE flag"
);
}
catch (IOException e)
{
GenericTestUtils.AssertExceptionContains("does not exist", e);
}
// Sanity test for CREATE
toAdd.Remove(generatedXAttrs[0]);
IList <XAttr> newXAttrs = FSDirXAttrOp.SetINodeXAttrs(fsdir, existingXAttrs, toAdd
, EnumSet.Of(XAttrSetFlag.Create));
NUnit.Framework.Assert.AreEqual("Unexpected toAdd size", 2, toAdd.Count);
foreach (XAttr x in toAdd)
{
NUnit.Framework.Assert.IsTrue("Did not find added XAttr " + x, newXAttrs.Contains
(x));
}
existingXAttrs = newXAttrs;
// Sanity test for REPLACE
toAdd = Lists.NewArrayList();
for (int i = 0; i < 3; i++)
{
XAttr xAttr = (new XAttr.Builder()).SetNameSpace(XAttr.NameSpace.System).SetName(
"a" + i).SetValue(new byte[] { unchecked ((byte)(i * 2)) }).Build();
toAdd.AddItem(xAttr);
}
newXAttrs = FSDirXAttrOp.SetINodeXAttrs(fsdir, existingXAttrs, toAdd, EnumSet.Of(
XAttrSetFlag.Replace));
NUnit.Framework.Assert.AreEqual("Unexpected number of new XAttrs", 3, newXAttrs.Count
);
for (int i_1 = 0; i_1 < 3; i_1++)
{
Assert.AssertArrayEquals("Unexpected XAttr value", new byte[] { unchecked ((byte)(
i_1 * 2)) }, newXAttrs[i_1].GetValue());
}
existingXAttrs = newXAttrs;
// Sanity test for CREATE+REPLACE
toAdd = Lists.NewArrayList();
for (int i_2 = 0; i_2 < 4; i_2++)
{
toAdd.AddItem(generatedXAttrs[i_2]);
}
newXAttrs = FSDirXAttrOp.SetINodeXAttrs(fsdir, existingXAttrs, toAdd, EnumSet.Of(
XAttrSetFlag.Create, XAttrSetFlag.Replace));
VerifyXAttrsPresent(newXAttrs, 4);
}
/// <summary>Build <code>XAttr</code> from name with prefix and value.</summary>
/// <remarks>
/// Build <code>XAttr</code> from name with prefix and value.
/// Name can not be null. Value can be null. The name and prefix
/// are validated.
/// Both name and namespace are case sensitive.
/// </remarks>
public static XAttr BuildXAttr(string name, byte[] value)
{
Preconditions.CheckNotNull(name, "XAttr name cannot be null.");
int prefixIndex = name.IndexOf(".");
if (prefixIndex < 3)
{
// Prefix length is at least 3.
throw new HadoopIllegalArgumentException("An XAttr name must be " + "prefixed with user/trusted/security/system/raw, followed by a '.'"
);
}
else
{
if (prefixIndex == name.Length - 1)
{
throw new HadoopIllegalArgumentException("XAttr name cannot be empty.");
}
}
XAttr.NameSpace ns;
string prefix = Sharpen.Runtime.Substring(name, 0, prefixIndex);
if (StringUtils.EqualsIgnoreCase(prefix, XAttr.NameSpace.User.ToString()))
{
ns = XAttr.NameSpace.User;
}
else
{
if (StringUtils.EqualsIgnoreCase(prefix, XAttr.NameSpace.Trusted.ToString()))
{
ns = XAttr.NameSpace.Trusted;
}
else
{
if (StringUtils.EqualsIgnoreCase(prefix, XAttr.NameSpace.System.ToString()))
{
ns = XAttr.NameSpace.System;
}
else
{
if (StringUtils.EqualsIgnoreCase(prefix, XAttr.NameSpace.Security.ToString()))
{
ns = XAttr.NameSpace.Security;
}
else
{
if (StringUtils.EqualsIgnoreCase(prefix, XAttr.NameSpace.Raw.ToString()))
{
ns = XAttr.NameSpace.Raw;
}
else
{
throw new HadoopIllegalArgumentException("An XAttr name must be " + "prefixed with user/trusted/security/system/raw, followed by a '.'"
);
}
}
}
}
}
XAttr xAttr = (new XAttr.Builder()).SetNameSpace(ns).SetName(Sharpen.Runtime.Substring
(name, prefixIndex + 1)).SetValue(value).Build();
return(xAttr);
}
public static bool IsStoragePolicyXAttr(XAttr xattr)
{
return(xattr != null && xattr.GetNameSpace() == XAttrNS && xattr.GetName().Equals
(StoragePolicyXattrName));
}
private static bool IsUserVisible(XAttr xAttr)
{
XAttr.NameSpace ns = xAttr.GetNameSpace();
return(ns == XAttr.NameSpace.User || ns == XAttr.NameSpace.Trusted);
}