// This is the only place that needs to use the BlockStoragePolicySuite to
// derive the intended storage type usage for quota by storage type
public sealed override QuotaCounts ComputeQuotaUsage(BlockStoragePolicySuite bsps
, byte blockStoragePolicyId, QuotaCounts counts, bool useCache, int lastSnapshotId
)
{
long nsDelta = 1;
long ssDeltaNoReplication;
short replication;
FileWithSnapshotFeature sf = GetFileWithSnapshotFeature();
if (sf != null)
{
FileDiffList fileDiffList = sf.GetDiffs();
int last = fileDiffList.GetLastSnapshotId();
if (lastSnapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId ||
last == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId)
{
ssDeltaNoReplication = StoragespaceConsumedNoReplication();
replication = GetBlockReplication();
}
else
{
if (last < lastSnapshotId)
{
ssDeltaNoReplication = ComputeFileSize(true, false);
replication = GetFileReplication();
}
else
{
int sid = fileDiffList.GetSnapshotById(lastSnapshotId);
ssDeltaNoReplication = StoragespaceConsumedNoReplication(sid);
replication = GetReplication(sid);
}
}
}
else
{
ssDeltaNoReplication = StoragespaceConsumedNoReplication();
replication = GetBlockReplication();
}
counts.AddNameSpace(nsDelta);
counts.AddStorageSpace(ssDeltaNoReplication * replication);
if (blockStoragePolicyId != BlockStoragePolicySuite.IdUnspecified)
{
BlockStoragePolicy bsp = bsps.GetPolicy(blockStoragePolicyId);
IList <StorageType> storageTypes = bsp.ChooseStorageTypes(replication);
foreach (StorageType t in storageTypes)
{
if (!t.SupportTypeQuota())
{
continue;
}
counts.AddTypeSpace(t, ssDeltaNoReplication);
}
}
return(counts);
}
private static QuotaCounts ComputeQuotaDeltas(FSDirectory fsd, INodeFile target,
INodeFile[] srcList)
{
QuotaCounts deltas = new QuotaCounts.Builder().Build();
short targetRepl = target.GetBlockReplication();
foreach (INodeFile src in srcList)
{
short srcRepl = src.GetBlockReplication();
long fileSize = src.ComputeFileSize();
if (targetRepl != srcRepl)
{
deltas.AddStorageSpace(fileSize * (targetRepl - srcRepl));
BlockStoragePolicy bsp = fsd.GetBlockStoragePolicySuite().GetPolicy(src.GetStoragePolicyID
());
if (bsp != null)
{
IList <StorageType> srcTypeChosen = bsp.ChooseStorageTypes(srcRepl);
foreach (StorageType t in srcTypeChosen)
{
if (t.SupportTypeQuota())
{
deltas.AddTypeSpace(t, -fileSize);
}
}
IList <StorageType> targetTypeChosen = bsp.ChooseStorageTypes(targetRepl);
foreach (StorageType t_1 in targetTypeChosen)
{
if (t_1.SupportTypeQuota())
{
deltas.AddTypeSpace(t_1, fileSize);
}
}
}
}
}
return(deltas);
}
public sealed override ContentSummaryComputationContext ComputeContentSummary(ContentSummaryComputationContext
summary)
{
ContentCounts counts = summary.GetCounts();
FileWithSnapshotFeature sf = GetFileWithSnapshotFeature();
long fileLen = 0;
if (sf == null)
{
fileLen = ComputeFileSize();
counts.AddContent(Content.File, 1);
}
else
{
FileDiffList diffs = sf.GetDiffs();
int n = diffs.AsList().Count;
counts.AddContent(Content.File, n);
if (n > 0 && sf.IsCurrentFileDeleted())
{
fileLen = diffs.GetLast().GetFileSize();
}
else
{
fileLen = ComputeFileSize();
}
}
counts.AddContent(Content.Length, fileLen);
counts.AddContent(Content.Diskspace, StoragespaceConsumed());
if (GetStoragePolicyID() != BlockStoragePolicySuite.IdUnspecified)
{
BlockStoragePolicy bsp = summary.GetBlockStoragePolicySuite().GetPolicy(GetStoragePolicyID
());
IList <StorageType> storageTypes = bsp.ChooseStorageTypes(GetFileReplication());
foreach (StorageType t in storageTypes)
{
if (!t.SupportTypeQuota())
{
continue;
}
counts.AddTypeSpace(t, fileLen);
}
}
return(summary);
}
/// <returns>true if it is necessary to run another round of migration</returns>
private bool ProcessFile(string fullPath, HdfsLocatedFileStatus status)
{
byte policyId = status.GetStoragePolicy();
// currently we ignore files with unspecified storage policy
if (policyId == BlockStoragePolicySuite.IdUnspecified)
{
return(false);
}
BlockStoragePolicy policy = this._enclosing.blockStoragePolicies[policyId];
if (policy == null)
{
Org.Apache.Hadoop.Hdfs.Server.Mover.Mover.Log.Warn("Failed to get the storage policy of file "
+ fullPath);
return(false);
}
IList <StorageType> types = policy.ChooseStorageTypes(status.GetReplication());
LocatedBlocks locatedBlocks = status.GetBlockLocations();
bool hasRemaining = false;
bool lastBlkComplete = locatedBlocks.IsLastBlockComplete();
IList <LocatedBlock> lbs = locatedBlocks.GetLocatedBlocks();
for (int i = 0; i < lbs.Count; i++)
{
if (i == lbs.Count - 1 && !lastBlkComplete)
{
// last block is incomplete, skip it
continue;
}
LocatedBlock lb = lbs[i];
Mover.StorageTypeDiff diff = new Mover.StorageTypeDiff(types, lb.GetStorageTypes(
));
if (!diff.RemoveOverlap(true))
{
if (this.ScheduleMoves4Block(diff, lb))
{
hasRemaining |= (diff.existing.Count > 1 && diff.expected.Count > 1);
}
}
}
return(hasRemaining);
}
/// <exception cref="System.Exception"/>
private void VerifyFile(Path parent, HdfsFileStatus status, byte expectedPolicyId
)
{
HdfsLocatedFileStatus fileStatus = (HdfsLocatedFileStatus)status;
byte policyId = fileStatus.GetStoragePolicy();
BlockStoragePolicy policy = this.policies.GetPolicy(policyId);
if (expectedPolicyId != null)
{
NUnit.Framework.Assert.AreEqual(unchecked ((byte)expectedPolicyId), policy.GetId()
);
}
IList <StorageType> types = policy.ChooseStorageTypes(status.GetReplication());
foreach (LocatedBlock lb in fileStatus.GetBlockLocations().GetLocatedBlocks())
{
Mover.StorageTypeDiff diff = new Mover.StorageTypeDiff(types, lb.GetStorageTypes(
));
NUnit.Framework.Assert.IsTrue(fileStatus.GetFullName(parent.ToString()) + " with policy "
+ policy + " has non-empty overlap: " + diff + ", the corresponding block is "
+ lb.GetBlock().GetLocalBlock(), diff.RemoveOverlap(true));
}
}
public virtual QuotaCounts UpdateQuotaAndCollectBlocks(BlockStoragePolicySuite bsps
, INodeFile file, FileDiff removed, INode.BlocksMapUpdateInfo collectedBlocks, IList
<INode> removedINodes)
{
long oldStoragespace = file.StoragespaceConsumed();
byte storagePolicyID = file.GetStoragePolicyID();
BlockStoragePolicy bsp = null;
EnumCounters <StorageType> typeSpaces = new EnumCounters <StorageType>(typeof(StorageType
));
if (storagePolicyID != BlockStoragePolicySuite.IdUnspecified)
{
bsp = bsps.GetPolicy(file.GetStoragePolicyID());
}
if (removed.snapshotINode != null)
{
short replication = removed.snapshotINode.GetFileReplication();
short currentRepl = file.GetBlockReplication();
if (currentRepl == 0)
{
long oldFileSizeNoRep = file.ComputeFileSize(true, true);
oldStoragespace = oldFileSizeNoRep * replication;
if (bsp != null)
{
IList <StorageType> oldTypeChosen = bsp.ChooseStorageTypes(replication);
foreach (StorageType t in oldTypeChosen)
{
if (t.SupportTypeQuota())
{
typeSpaces.Add(t, -oldFileSizeNoRep);
}
}
}
}
else
{
if (replication > currentRepl)
{
long oldFileSizeNoRep = file.StoragespaceConsumedNoReplication();
oldStoragespace = oldFileSizeNoRep * replication;
if (bsp != null)
{
IList <StorageType> oldTypeChosen = bsp.ChooseStorageTypes(replication);
foreach (StorageType t in oldTypeChosen)
{
if (t.SupportTypeQuota())
{
typeSpaces.Add(t, -oldFileSizeNoRep);
}
}
IList <StorageType> newTypeChosen = bsp.ChooseStorageTypes(currentRepl);
foreach (StorageType t_1 in newTypeChosen)
{
if (t_1.SupportTypeQuota())
{
typeSpaces.Add(t_1, oldFileSizeNoRep);
}
}
}
}
}
AclFeature aclFeature = removed.GetSnapshotINode().GetAclFeature();
if (aclFeature != null)
{
AclStorage.RemoveAclFeature(aclFeature);
}
}
GetDiffs().CombineAndCollectSnapshotBlocks(bsps, file, removed, collectedBlocks,
removedINodes);
long ssDelta = oldStoragespace - file.StoragespaceConsumed();
return(new QuotaCounts.Builder().StorageSpace(ssDelta).TypeSpaces(typeSpaces).Build
());
}
