/// <summary>
/// If some blocks at the end of the block list no longer belongs to
/// any inode, collect them and update the block list.
/// </summary>
public virtual void CollectBlocksAndClear(BlockStoragePolicySuite bsps, INodeFile
file, INode.BlocksMapUpdateInfo info, IList <INode> removedINodes)
{
// check if everything is deleted.
if (IsCurrentFileDeleted() && GetDiffs().AsList().IsEmpty())
{
file.DestroyAndCollectBlocks(bsps, info, removedINodes);
return;
}
// find max file size.
long max;
FileDiff diff = GetDiffs().GetLast();
if (IsCurrentFileDeleted())
{
max = diff == null ? 0 : diff.GetFileSize();
}
else
{
max = file.ComputeFileSize();
}
// Collect blocks that should be deleted
FileDiff last = diffs.GetLast();
BlockInfoContiguous[] snapshotBlocks = last == null ? null : last.GetBlocks();
if (snapshotBlocks == null)
{
file.CollectBlocksBeyondMax(max, info);
}
else
{
file.CollectBlocksBeyondSnapshot(snapshotBlocks, info);
}
}
/// <exception cref="System.IO.IOException"/>
private void SerializeFileDiffList(INodeFile file, OutputStream @out)
{
FileWithSnapshotFeature sf = file.GetFileWithSnapshotFeature();
if (sf != null)
{
IList <FileDiff> diffList = sf.GetDiffs().AsList();
FsImageProto.SnapshotDiffSection.DiffEntry entry = ((FsImageProto.SnapshotDiffSection.DiffEntry
)FsImageProto.SnapshotDiffSection.DiffEntry.NewBuilder().SetInodeId(file.GetId()
).SetType(FsImageProto.SnapshotDiffSection.DiffEntry.Type.Filediff).SetNumOfDiff
(diffList.Count).Build());
entry.WriteDelimitedTo(@out);
for (int i = diffList.Count - 1; i >= 0; i--)
{
FileDiff diff = diffList[i];
FsImageProto.SnapshotDiffSection.FileDiff.Builder fb = FsImageProto.SnapshotDiffSection.FileDiff
.NewBuilder().SetSnapshotId(diff.GetSnapshotId()).SetFileSize(diff.GetFileSize()
);
if (diff.GetBlocks() != null)
{
foreach (Block block in diff.GetBlocks())
{
fb.AddBlocks(PBHelper.Convert(block));
}
}
INodeFileAttributes copy = diff.snapshotINode;
if (copy != null)
{
fb.SetName(ByteString.CopyFrom(copy.GetLocalNameBytes())).SetSnapshotCopy(FSImageFormatPBINode.Saver.BuildINodeFile
(copy, parent.GetSaverContext()));
}
((FsImageProto.SnapshotDiffSection.FileDiff)fb.Build()).WriteDelimitedTo(@out);
}
}
}
/// <summary>Load FileDiff list for a file with snapshot feature</summary>
/// <exception cref="System.IO.IOException"/>
private void LoadFileDiffList(InputStream @in, INodeFile file, int size)
{
FileDiffList diffs = new FileDiffList();
FSImageFormatProtobuf.LoaderContext state = parent.GetLoaderContext();
for (int i = 0; i < size; i++)
{
FsImageProto.SnapshotDiffSection.FileDiff pbf = FsImageProto.SnapshotDiffSection.FileDiff
.ParseDelimitedFrom(@in);
INodeFileAttributes copy = null;
if (pbf.HasSnapshotCopy())
{
FsImageProto.INodeSection.INodeFile fileInPb = pbf.GetSnapshotCopy();
PermissionStatus permission = FSImageFormatPBINode.Loader.LoadPermission(fileInPb
.GetPermission(), state.GetStringTable());
AclFeature acl = null;
if (fileInPb.HasAcl())
{
int[] entries = AclEntryStatusFormat.ToInt(FSImageFormatPBINode.Loader.LoadAclEntries
(fileInPb.GetAcl(), state.GetStringTable()));
acl = new AclFeature(entries);
}
XAttrFeature xAttrs = null;
if (fileInPb.HasXAttrs())
{
xAttrs = new XAttrFeature(FSImageFormatPBINode.Loader.LoadXAttrs(fileInPb.GetXAttrs
(), state.GetStringTable()));
}
copy = new INodeFileAttributes.SnapshotCopy(pbf.GetName().ToByteArray(), permission
, acl, fileInPb.GetModificationTime(), fileInPb.GetAccessTime(), (short)fileInPb
.GetReplication(), fileInPb.GetPreferredBlockSize(), unchecked ((byte)fileInPb.GetStoragePolicyID
()), xAttrs);
}
FileDiff diff = new FileDiff(pbf.GetSnapshotId(), copy, null, pbf.GetFileSize());
IList <HdfsProtos.BlockProto> bpl = pbf.GetBlocksList();
BlockInfoContiguous[] blocks = new BlockInfoContiguous[bpl.Count];
for (int j = 0; j < e; ++j)
{
Block blk = PBHelper.Convert(bpl[j]);
BlockInfoContiguous storedBlock = fsn.GetBlockManager().GetStoredBlock(blk);
if (storedBlock == null)
{
storedBlock = fsn.GetBlockManager().AddBlockCollection(new BlockInfoContiguous(blk
, copy.GetFileReplication()), file);
}
blocks[j] = storedBlock;
}
if (blocks.Length > 0)
{
diff.SetBlocks(blocks);
}
diffs.AddFirst(diff);
}
file.AddSnapshotFeature(diffs);
}
/// <exception cref="System.IO.IOException"/>
private static FileDiff LoadFileDiff(FileDiff posterior, DataInput @in, FSImageFormat.Loader
loader)
{
// 1. Read the id of the Snapshot root to identify the Snapshot
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot snapshot = loader.GetSnapshot
(@in);
// 2. Load file size
long fileSize = @in.ReadLong();
// 3. Load snapshotINode
INodeFileAttributes snapshotINode = @in.ReadBoolean() ? loader.LoadINodeFileAttributes
(@in) : null;
return(new FileDiff(snapshot.GetId(), snapshotINode, posterior, fileSize));
}
internal virtual bool ChangedBetweenSnapshots(INodeFile file, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
from, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot to)
{
int[] diffIndexPair = diffs.ChangedBetweenSnapshots(from, to);
if (diffIndexPair == null)
{
return(false);
}
int earlierDiffIndex = diffIndexPair[0];
int laterDiffIndex = diffIndexPair[1];
IList <FileDiff> diffList = diffs.AsList();
long earlierLength = diffList[earlierDiffIndex].GetFileSize();
long laterLength = laterDiffIndex == diffList.Count ? file.ComputeFileSize(true,
false) : diffList[laterDiffIndex].GetFileSize();
if (earlierLength != laterLength)
{
// file length has been changed
return(true);
}
INodeFileAttributes earlierAttr = null;
// check the metadata
for (int i = earlierDiffIndex; i < laterDiffIndex; i++)
{
FileDiff diff = diffList[i];
if (diff.snapshotINode != null)
{
earlierAttr = diff.snapshotINode;
break;
}
}
if (earlierAttr == null)
{
// no meta-change at all, return false
return(false);
}
INodeFileAttributes laterAttr = diffs.GetSnapshotINode(Math.Max(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.GetSnapshotId(from), Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.GetSnapshotId
(to)), file);
return(!earlierAttr.MetadataEquals(laterAttr));
}
/// <exception cref="System.IO.IOException"/>
public static FileDiffList LoadFileDiffList(DataInput @in, FSImageFormat.Loader loader
)
{
int size = @in.ReadInt();
if (size == -1)
{
return(null);
}
else
{
FileDiffList diffs = new FileDiffList();
FileDiff posterior = null;
for (int i = 0; i < size; i++)
{
FileDiff d = LoadFileDiff(posterior, @in, loader);
diffs.AddFirst(d);
posterior = d;
}
return(diffs);
}
}
public virtual QuotaCounts UpdateQuotaAndCollectBlocks(BlockStoragePolicySuite bsps
, INodeFile file, FileDiff removed, INode.BlocksMapUpdateInfo collectedBlocks, IList
<INode> removedINodes)
{
long oldStoragespace = file.StoragespaceConsumed();
byte storagePolicyID = file.GetStoragePolicyID();
BlockStoragePolicy bsp = null;
EnumCounters <StorageType> typeSpaces = new EnumCounters <StorageType>(typeof(StorageType
));
if (storagePolicyID != BlockStoragePolicySuite.IdUnspecified)
{
bsp = bsps.GetPolicy(file.GetStoragePolicyID());
}
if (removed.snapshotINode != null)
{
short replication = removed.snapshotINode.GetFileReplication();
short currentRepl = file.GetBlockReplication();
if (currentRepl == 0)
{
long oldFileSizeNoRep = file.ComputeFileSize(true, true);
oldStoragespace = oldFileSizeNoRep * replication;
if (bsp != null)
{
IList <StorageType> oldTypeChosen = bsp.ChooseStorageTypes(replication);
foreach (StorageType t in oldTypeChosen)
{
if (t.SupportTypeQuota())
{
typeSpaces.Add(t, -oldFileSizeNoRep);
}
}
}
}
else
{
if (replication > currentRepl)
{
long oldFileSizeNoRep = file.StoragespaceConsumedNoReplication();
oldStoragespace = oldFileSizeNoRep * replication;
if (bsp != null)
{
IList <StorageType> oldTypeChosen = bsp.ChooseStorageTypes(replication);
foreach (StorageType t in oldTypeChosen)
{
if (t.SupportTypeQuota())
{
typeSpaces.Add(t, -oldFileSizeNoRep);
}
}
IList <StorageType> newTypeChosen = bsp.ChooseStorageTypes(currentRepl);
foreach (StorageType t_1 in newTypeChosen)
{
if (t_1.SupportTypeQuota())
{
typeSpaces.Add(t_1, oldFileSizeNoRep);
}
}
}
}
}
AclFeature aclFeature = removed.GetSnapshotINode().GetAclFeature();
if (aclFeature != null)
{
AclStorage.RemoveAclFeature(aclFeature);
}
}
GetDiffs().CombineAndCollectSnapshotBlocks(bsps, file, removed, collectedBlocks,
removedINodes);
long ssDelta = oldStoragespace - file.StoragespaceConsumed();
return(new QuotaCounts.Builder().StorageSpace(ssDelta).TypeSpaces(typeSpaces).Build
());
}
