/// <summary>Create FileStatus with location info by file INode</summary> /// <exception cref="System.IO.IOException"/> private static HdfsLocatedFileStatus CreateLocatedFileStatus(FSDirectory fsd, string fullPath, byte[] path, INode node, byte storagePolicy, int snapshot, bool isRawPath , INodesInPath iip) { System.Diagnostics.Debug.Assert(fsd.HasReadLock()); long size = 0; // length is zero for directories short replication = 0; long blocksize = 0; LocatedBlocks loc = null; bool isEncrypted; FileEncryptionInfo feInfo = isRawPath ? null : fsd.GetFileEncryptionInfo(node, snapshot , iip); if (node.IsFile()) { INodeFile fileNode = node.AsFile(); size = fileNode.ComputeFileSize(snapshot); replication = fileNode.GetFileReplication(snapshot); blocksize = fileNode.GetPreferredBlockSize(); bool inSnapshot = snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .CurrentStateId; bool isUc = !inSnapshot && fileNode.IsUnderConstruction(); long fileSize = !inSnapshot && isUc?fileNode.ComputeFileSizeNotIncludingLastUcBlock () : size; loc = fsd.GetFSNamesystem().GetBlockManager().CreateLocatedBlocks(fileNode.GetBlocks (snapshot), fileSize, isUc, 0L, size, false, inSnapshot, feInfo); if (loc == null) { loc = new LocatedBlocks(); } isEncrypted = (feInfo != null) || (isRawPath && fsd.IsInAnEZ(INodesInPath.FromINode (node))); } else { isEncrypted = fsd.IsInAnEZ(INodesInPath.FromINode(node)); } int childrenNum = node.IsDirectory() ? node.AsDirectory().GetChildrenNum(snapshot ) : 0; INodeAttributes nodeAttrs = fsd.GetAttributes(fullPath, path, node, snapshot); HdfsLocatedFileStatus status = new HdfsLocatedFileStatus(size, node.IsDirectory() , replication, blocksize, node.GetModificationTime(snapshot), node.GetAccessTime (snapshot), GetPermissionForFileStatus(nodeAttrs, isEncrypted), nodeAttrs.GetUserName (), nodeAttrs.GetGroupName(), node.IsSymlink() ? node.AsSymlink().GetSymlink() : null, path, node.GetId(), loc, childrenNum, feInfo, storagePolicy); // Set caching information for the located blocks. if (loc != null) { CacheManager cacheManager = fsd.GetFSNamesystem().GetCacheManager(); foreach (LocatedBlock lb in loc.GetLocatedBlocks()) { cacheManager.SetCachedLocations(lb); } } return(status); }
/// <returns>true if it is necessary to run another round of migration</returns> private bool ProcessFile(string fullPath, HdfsLocatedFileStatus status) { byte policyId = status.GetStoragePolicy(); // currently we ignore files with unspecified storage policy if (policyId == BlockStoragePolicySuite.IdUnspecified) { return(false); } BlockStoragePolicy policy = this._enclosing.blockStoragePolicies[policyId]; if (policy == null) { Org.Apache.Hadoop.Hdfs.Server.Mover.Mover.Log.Warn("Failed to get the storage policy of file " + fullPath); return(false); } IList <StorageType> types = policy.ChooseStorageTypes(status.GetReplication()); LocatedBlocks locatedBlocks = status.GetBlockLocations(); bool hasRemaining = false; bool lastBlkComplete = locatedBlocks.IsLastBlockComplete(); IList <LocatedBlock> lbs = locatedBlocks.GetLocatedBlocks(); for (int i = 0; i < lbs.Count; i++) { if (i == lbs.Count - 1 && !lastBlkComplete) { // last block is incomplete, skip it continue; } LocatedBlock lb = lbs[i]; Mover.StorageTypeDiff diff = new Mover.StorageTypeDiff(types, lb.GetStorageTypes( )); if (!diff.RemoveOverlap(true)) { if (this.ScheduleMoves4Block(diff, lb)) { hasRemaining |= (diff.existing.Count > 1 && diff.expected.Count > 1); } } } return(hasRemaining); }
/// <exception cref="System.Exception"/> private void VerifyFile(Path parent, HdfsFileStatus status, byte expectedPolicyId ) { HdfsLocatedFileStatus fileStatus = (HdfsLocatedFileStatus)status; byte policyId = fileStatus.GetStoragePolicy(); BlockStoragePolicy policy = this.policies.GetPolicy(policyId); if (expectedPolicyId != null) { NUnit.Framework.Assert.AreEqual(unchecked ((byte)expectedPolicyId), policy.GetId() ); } IList <StorageType> types = policy.ChooseStorageTypes(status.GetReplication()); foreach (LocatedBlock lb in fileStatus.GetBlockLocations().GetLocatedBlocks()) { Mover.StorageTypeDiff diff = new Mover.StorageTypeDiff(types, lb.GetStorageTypes( )); NUnit.Framework.Assert.IsTrue(fileStatus.GetFullName(parent.ToString()) + " with policy " + policy + " has non-empty overlap: " + diff + ", the corresponding block is " + lb.GetBlock().GetLocalBlock(), diff.RemoveOverlap(true)); } }