public virtual void TestCommitBlockSynchronization()
{
INodeFile file = MockFileUnderConstruction();
Block block = new Block(blockId, length, genStamp);
FSNamesystem namesystemSpy = MakeNameSystemSpy(block, file);
DatanodeID[] newTargets = new DatanodeID[0];
ExtendedBlock lastBlock = new ExtendedBlock();
namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false
, newTargets, null);
// Repeat the call to make sure it does not throw
namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false
, newTargets, null);
// Simulate 'completing' the block.
BlockInfoContiguous completedBlockInfo = new BlockInfoContiguous(block, (short)1);
completedBlockInfo.SetBlockCollection(file);
completedBlockInfo.SetGenerationStamp(genStamp);
Org.Mockito.Mockito.DoReturn(completedBlockInfo).When(namesystemSpy).GetStoredBlock
(Matchers.Any <Block>());
Org.Mockito.Mockito.DoReturn(completedBlockInfo).When(file).GetLastBlock();
// Repeat the call to make sure it does not throw
namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false
, newTargets, null);
}
public long StoragespaceConsumedNoReplication()
{
FileWithSnapshotFeature sf = GetFileWithSnapshotFeature();
if (sf == null)
{
return(ComputeFileSize(true, true));
}
// Collect all distinct blocks
long size = 0;
ICollection <Block> allBlocks = new HashSet <Block>(Arrays.AsList(GetBlocks()));
IList <FileDiff> diffs = sf.GetDiffs().AsList();
foreach (FileDiff diff in diffs)
{
BlockInfoContiguous[] diffBlocks = diff.GetBlocks();
if (diffBlocks != null)
{
Sharpen.Collections.AddAll(allBlocks, Arrays.AsList(diffBlocks));
}
}
foreach (Block block in allBlocks)
{
size += block.GetNumBytes();
}
// check if the last block is under construction
BlockInfoContiguous lastBlock = GetLastBlock();
if (lastBlock != null && lastBlock is BlockInfoContiguousUnderConstruction)
{
size += GetPreferredBlockSize() - lastBlock.GetNumBytes();
}
return(size);
}
internal static void AssertBlockCollection(BlockManager blkManager, INodeFile file
, BlockInfoContiguous b)
{
NUnit.Framework.Assert.AreSame(b, blkManager.GetStoredBlock(b));
NUnit.Framework.Assert.AreSame(file, blkManager.GetBlockCollection(b));
NUnit.Framework.Assert.AreSame(file, b.GetBlockCollection());
}
internal static void AddFiles(FSEditLog editLog, int numFiles, short replication,
int blocksPerFile, long startingBlockId, long blockSize, FileNameGenerator nameGenerator
)
{
PermissionStatus p = new PermissionStatus("joeDoe", "people", new FsPermission((short
)0x1ff));
INodeId inodeId = new INodeId();
INodeDirectory dirInode = new INodeDirectory(inodeId.NextValue(), null, p, 0L);
editLog.LogMkDir(BasePath, dirInode);
BlockInfoContiguous[] blocks = new BlockInfoContiguous[blocksPerFile];
for (int iB = 0; iB < blocksPerFile; ++iB)
{
blocks[iB] = new BlockInfoContiguous(new Block(0, blockSize, BlockGenerationStamp
), replication);
}
long currentBlockId = startingBlockId;
long bidAtSync = startingBlockId;
for (int iF = 0; iF < numFiles; iF++)
{
for (int iB_1 = 0; iB_1 < blocksPerFile; ++iB_1)
{
blocks[iB_1].SetBlockId(currentBlockId++);
}
INodeFile inode = new INodeFile(inodeId.NextValue(), null, p, 0L, 0L, blocks, replication
, blockSize, unchecked ((byte)0));
inode.ToUnderConstruction(string.Empty, string.Empty);
// Append path to filename with information about blockIDs
string path = "_" + iF + "_B" + blocks[0].GetBlockId() + "_to_B" + blocks[blocksPerFile
- 1].GetBlockId() + "_";
string filePath = nameGenerator.GetNextFileName(string.Empty);
filePath = filePath + path;
// Log the new sub directory in edits
if ((iF % nameGenerator.GetFilesPerDirectory()) == 0)
{
string currentDir = nameGenerator.GetCurrentDir();
dirInode = new INodeDirectory(inodeId.NextValue(), null, p, 0L);
editLog.LogMkDir(currentDir, dirInode);
}
INodeFile fileUc = new INodeFile(inodeId.NextValue(), null, p, 0L, 0L, BlockInfoContiguous
.EmptyArray, replication, blockSize);
fileUc.ToUnderConstruction(string.Empty, string.Empty);
editLog.LogOpenFile(filePath, fileUc, false, false);
editLog.LogCloseFile(filePath, inode);
if (currentBlockId - bidAtSync >= 2000)
{
// sync every 2K blocks
editLog.LogSync();
bidAtSync = currentBlockId;
}
}
System.Console.Out.WriteLine("Created edits log in directory " + edits_dir);
System.Console.Out.WriteLine(" containing " + numFiles + " File-Creates, each file with "
+ blocksPerFile + " blocks");
System.Console.Out.WriteLine(" blocks range: " + startingBlockId + " to " + (currentBlockId
- 1));
}
/// <summary>Update the length for the last block</summary>
/// <param name="lastBlockLength">The length of the last block reported from client</param>
/// <exception cref="System.IO.IOException"/>
internal virtual void UpdateLengthOfLastBlock(INodeFile f, long lastBlockLength)
{
BlockInfoContiguous lastBlock = f.GetLastBlock();
System.Diagnostics.Debug.Assert((lastBlock != null), "The last block for path " +
f.GetFullPathName() + " is null when updating its length");
System.Diagnostics.Debug.Assert((lastBlock is BlockInfoContiguousUnderConstruction
), "The last block for path " + f.GetFullPathName() + " is not a BlockInfoUnderConstruction when updating its length"
);
lastBlock.SetNumBytes(lastBlockLength);
}
/// <summary>Load FileDiff list for a file with snapshot feature</summary>
/// <exception cref="System.IO.IOException"/>
private void LoadFileDiffList(InputStream @in, INodeFile file, int size)
{
FileDiffList diffs = new FileDiffList();
FSImageFormatProtobuf.LoaderContext state = parent.GetLoaderContext();
for (int i = 0; i < size; i++)
{
FsImageProto.SnapshotDiffSection.FileDiff pbf = FsImageProto.SnapshotDiffSection.FileDiff
.ParseDelimitedFrom(@in);
INodeFileAttributes copy = null;
if (pbf.HasSnapshotCopy())
{
FsImageProto.INodeSection.INodeFile fileInPb = pbf.GetSnapshotCopy();
PermissionStatus permission = FSImageFormatPBINode.Loader.LoadPermission(fileInPb
.GetPermission(), state.GetStringTable());
AclFeature acl = null;
if (fileInPb.HasAcl())
{
int[] entries = AclEntryStatusFormat.ToInt(FSImageFormatPBINode.Loader.LoadAclEntries
(fileInPb.GetAcl(), state.GetStringTable()));
acl = new AclFeature(entries);
}
XAttrFeature xAttrs = null;
if (fileInPb.HasXAttrs())
{
xAttrs = new XAttrFeature(FSImageFormatPBINode.Loader.LoadXAttrs(fileInPb.GetXAttrs
(), state.GetStringTable()));
}
copy = new INodeFileAttributes.SnapshotCopy(pbf.GetName().ToByteArray(), permission
, acl, fileInPb.GetModificationTime(), fileInPb.GetAccessTime(), (short)fileInPb
.GetReplication(), fileInPb.GetPreferredBlockSize(), unchecked ((byte)fileInPb.GetStoragePolicyID
()), xAttrs);
}
FileDiff diff = new FileDiff(pbf.GetSnapshotId(), copy, null, pbf.GetFileSize());
IList <HdfsProtos.BlockProto> bpl = pbf.GetBlocksList();
BlockInfoContiguous[] blocks = new BlockInfoContiguous[bpl.Count];
for (int j = 0; j < e; ++j)
{
Block blk = PBHelper.Convert(bpl[j]);
BlockInfoContiguous storedBlock = fsn.GetBlockManager().GetStoredBlock(blk);
if (storedBlock == null)
{
storedBlock = fsn.GetBlockManager().AddBlockCollection(new BlockInfoContiguous(blk
, copy.GetFileReplication()), file);
}
blocks[j] = storedBlock;
}
if (blocks.Length > 0)
{
diff.SetBlocks(blocks);
}
diffs.AddFirst(diff);
}
file.AddSnapshotFeature(diffs);
}
/// <returns>true if the block is contained in a snapshot or false otherwise.</returns>
internal virtual bool IsBlockInLatestSnapshot(BlockInfoContiguous block)
{
FileWithSnapshotFeature sf = this.GetFileWithSnapshotFeature();
if (sf == null || sf.GetDiffs() == null)
{
return(false);
}
BlockInfoContiguous[] snapshotBlocks = GetDiffs().FindEarlierSnapshotBlocks(GetDiffs
().GetLastSnapshotId());
return(snapshotBlocks != null && Arrays.AsList(snapshotBlocks).Contains(block));
}
internal virtual void TruncateBlocksTo(int n)
{
BlockInfoContiguous[] newBlocks;
if (n == 0)
{
newBlocks = BlockInfoContiguous.EmptyArray;
}
else
{
newBlocks = new BlockInfoContiguous[n];
System.Array.Copy(GetBlocks(), 0, newBlocks, 0, n);
}
// set new blocks
SetBlocks(newBlocks);
}
/// <summary>add a block to the block list</summary>
internal virtual void AddBlock(BlockInfoContiguous newblock)
{
if (this.blocks == null)
{
this.SetBlocks(new BlockInfoContiguous[] { newblock });
}
else
{
int size = this.blocks.Length;
BlockInfoContiguous[] newlist = new BlockInfoContiguous[size + 1];
System.Array.Copy(this.blocks, 0, newlist, 0, size);
newlist[size] = newblock;
this.SetBlocks(newlist);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual BlockInfoContiguousUnderConstruction SetLastBlock(BlockInfoContiguous
lastBlock, DatanodeStorageInfo[] locations)
{
// BlockCollection, the file should be under construction
Preconditions.CheckState(IsUnderConstruction(), "file is no longer under construction"
);
if (NumBlocks() == 0)
{
throw new IOException("Failed to set last block: File is empty.");
}
BlockInfoContiguousUnderConstruction ucBlock = lastBlock.ConvertToBlockUnderConstruction
(HdfsServerConstants.BlockUCState.UnderConstruction, locations);
SetBlock(NumBlocks() - 1, ucBlock);
return(ucBlock);
}
private INodeFile LoadINodeFile(FsImageProto.INodeSection.INode n)
{
System.Diagnostics.Debug.Assert(n.GetType() == FsImageProto.INodeSection.INode.Type
.File);
FsImageProto.INodeSection.INodeFile f = n.GetFile();
IList <HdfsProtos.BlockProto> bp = f.GetBlocksList();
short replication = (short)f.GetReplication();
FSImageFormatProtobuf.LoaderContext state = parent.GetLoaderContext();
BlockInfoContiguous[] blocks = new BlockInfoContiguous[bp.Count];
for (int i = 0; i < e; ++i)
{
blocks[i] = new BlockInfoContiguous(PBHelper.Convert(bp[i]), replication);
}
PermissionStatus permissions = LoadPermission(f.GetPermission(), parent.GetLoaderContext
().GetStringTable());
INodeFile file = new INodeFile(n.GetId(), n.GetName().ToByteArray(), permissions,
f.GetModificationTime(), f.GetAccessTime(), blocks, replication, f.GetPreferredBlockSize
(), unchecked ((byte)f.GetStoragePolicyID()));
if (f.HasAcl())
{
int[] entries = AclEntryStatusFormat.ToInt(LoadAclEntries(f.GetAcl(), state.GetStringTable
()));
file.AddAclFeature(new AclFeature(entries));
}
if (f.HasXAttrs())
{
file.AddXAttrFeature(new XAttrFeature(LoadXAttrs(f.GetXAttrs(), state.GetStringTable
())));
}
// under-construction information
if (f.HasFileUC())
{
FsImageProto.INodeSection.FileUnderConstructionFeature uc = f.GetFileUC();
file.ToUnderConstruction(uc.GetClientName(), uc.GetClientMachine());
if (blocks.Length > 0)
{
BlockInfoContiguous lastBlk = file.GetLastBlock();
// replace the last block of file
file.SetBlock(file.NumBlocks() - 1, new BlockInfoContiguousUnderConstruction(lastBlk
, replication));
}
}
return(file);
}
/// <summary>append array of blocks to this.blocks</summary>
internal virtual void ConcatBlocks(INodeFile[] inodes)
{
int size = this.blocks.Length;
int totalAddedBlocks = 0;
foreach (INodeFile f in inodes)
{
totalAddedBlocks += f.blocks.Length;
}
BlockInfoContiguous[] newlist = new BlockInfoContiguous[size + totalAddedBlocks];
System.Array.Copy(this.blocks, 0, newlist, 0, size);
foreach (INodeFile @in in inodes)
{
System.Array.Copy(@in.blocks, 0, newlist, size, @in.blocks.Length);
size += @in.blocks.Length;
}
SetBlocks(newlist);
UpdateBlockCollection();
}
// Helper function that reads in an INodeUnderConstruction
// from the input stream
//
/// <exception cref="System.IO.IOException"/>
internal static INodeFile ReadINodeUnderConstruction(DataInput @in, FSNamesystem
fsNamesys, int imgVersion)
{
byte[] name = ReadBytes(@in);
long inodeId = NameNodeLayoutVersion.Supports(LayoutVersion.Feature.AddInodeId, imgVersion
) ? @in.ReadLong() : fsNamesys.dir.AllocateNewInodeId();
short blockReplication = @in.ReadShort();
long modificationTime = @in.ReadLong();
long preferredBlockSize = @in.ReadLong();
int numBlocks = @in.ReadInt();
BlockInfoContiguous[] blocks = new BlockInfoContiguous[numBlocks];
Block blk = new Block();
int i = 0;
for (; i < numBlocks - 1; i++)
{
blk.ReadFields(@in);
blocks[i] = new BlockInfoContiguous(blk, blockReplication);
}
// last block is UNDER_CONSTRUCTION
if (numBlocks > 0)
{
blk.ReadFields(@in);
blocks[i] = new BlockInfoContiguousUnderConstruction(blk, blockReplication, HdfsServerConstants.BlockUCState
.UnderConstruction, null);
}
PermissionStatus perm = PermissionStatus.Read(@in);
string clientName = ReadString(@in);
string clientMachine = ReadString(@in);
// We previously stored locations for the last block, now we
// just record that there are none
int numLocs = @in.ReadInt();
System.Diagnostics.Debug.Assert(numLocs == 0, "Unexpected block locations");
// Images in the pre-protobuf format will not have the lazyPersist flag,
// so it is safe to pass false always.
INodeFile file = new INodeFile(inodeId, name, perm, modificationTime, modificationTime
, blocks, blockReplication, preferredBlockSize, unchecked ((byte)0));
file.ToUnderConstruction(clientName, clientMachine);
return(file);
}
/// <summary>Remove a block from the block list.</summary>
/// <remarks>
/// Remove a block from the block list. This block should be
/// the last one on the list.
/// </remarks>
internal virtual bool RemoveLastBlock(Block oldblock)
{
Preconditions.CheckState(IsUnderConstruction(), "file is no longer under construction"
);
if (blocks == null || blocks.Length == 0)
{
return(false);
}
int size_1 = blocks.Length - 1;
if (!blocks[size_1].Equals(oldblock))
{
return(false);
}
//copy to a new list
BlockInfoContiguous[] newlist = new BlockInfoContiguous[size_1];
System.Array.Copy(blocks, 0, newlist, 0, size_1);
SetBlocks(newlist);
return(true);
}
public virtual void SetBlock(int index, BlockInfoContiguous blk)
{
// BlockCollection
this.blocks[index] = blk;
}
/// <summary>
/// Copy blocks from the removed snapshot into the previous snapshot
/// up to the file length of the latter.
/// </summary>
/// <remarks>
/// Copy blocks from the removed snapshot into the previous snapshot
/// up to the file length of the latter.
/// Collect unused blocks of the removed snapshot.
/// </remarks>
internal virtual void CombineAndCollectSnapshotBlocks(BlockStoragePolicySuite bsps
, INodeFile file, FileDiff removed, INode.BlocksMapUpdateInfo collectedBlocks, IList
<INode> removedINodes)
{
BlockInfoContiguous[] removedBlocks = removed.GetBlocks();
if (removedBlocks == null)
{
FileWithSnapshotFeature sf = file.GetFileWithSnapshotFeature();
System.Diagnostics.Debug.Assert(sf != null, "FileWithSnapshotFeature is null");
if (sf.IsCurrentFileDeleted())
{
sf.CollectBlocksAndClear(bsps, file, collectedBlocks, removedINodes);
}
return;
}
int p = GetPrior(removed.GetSnapshotId(), true);
FileDiff earlierDiff = p == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.NoSnapshotId ? null : GetDiffById(p);
// Copy blocks to the previous snapshot if not set already
if (earlierDiff != null)
{
earlierDiff.SetBlocks(removedBlocks);
}
BlockInfoContiguous[] earlierBlocks = (earlierDiff == null ? new BlockInfoContiguous
[] { } : earlierDiff.GetBlocks());
// Find later snapshot (or file itself) with blocks
BlockInfoContiguous[] laterBlocks = FindLaterSnapshotBlocks(removed.GetSnapshotId
());
laterBlocks = (laterBlocks == null) ? file.GetBlocks() : laterBlocks;
// Skip blocks, which belong to either the earlier or the later lists
int i = 0;
for (; i < removedBlocks.Length; i++)
{
if (i < earlierBlocks.Length && removedBlocks[i] == earlierBlocks[i])
{
continue;
}
if (i < laterBlocks.Length && removedBlocks[i] == laterBlocks[i])
{
continue;
}
break;
}
// Check if last block is part of truncate recovery
BlockInfoContiguous lastBlock = file.GetLastBlock();
Block dontRemoveBlock = null;
if (lastBlock != null && lastBlock.GetBlockUCState().Equals(HdfsServerConstants.BlockUCState
.UnderRecovery))
{
dontRemoveBlock = ((BlockInfoContiguousUnderConstruction)lastBlock).GetTruncateBlock
();
}
// Collect the remaining blocks of the file, ignoring truncate block
for (; i < removedBlocks.Length; i++)
{
if (dontRemoveBlock == null || !removedBlocks[i].Equals(dontRemoveBlock))
{
collectedBlocks.AddDeleteBlock(removedBlocks[i]);
}
}
}
