/// <summary>Create FileStatus by file INode</summary> /// <exception cref="System.IO.IOException"/> internal static HdfsFileStatus CreateFileStatus(FSDirectory fsd, string fullPath, byte[] path, INode node, byte storagePolicy, int snapshot, bool isRawPath, INodesInPath iip) { long size = 0; // length is zero for directories short replication = 0; long blocksize = 0; bool isEncrypted; FileEncryptionInfo feInfo = isRawPath ? null : fsd.GetFileEncryptionInfo(node, snapshot , iip); if (node.IsFile()) { INodeFile fileNode = node.AsFile(); size = fileNode.ComputeFileSize(snapshot); replication = fileNode.GetFileReplication(snapshot); blocksize = fileNode.GetPreferredBlockSize(); isEncrypted = (feInfo != null) || (isRawPath && fsd.IsInAnEZ(INodesInPath.FromINode (node))); } else { isEncrypted = fsd.IsInAnEZ(INodesInPath.FromINode(node)); } int childrenNum = node.IsDirectory() ? node.AsDirectory().GetChildrenNum(snapshot ) : 0; INodeAttributes nodeAttrs = fsd.GetAttributes(fullPath, path, node, snapshot); return(new HdfsFileStatus(size, node.IsDirectory(), replication, blocksize, node. GetModificationTime(snapshot), node.GetAccessTime(snapshot), GetPermissionForFileStatus (nodeAttrs, isEncrypted), nodeAttrs.GetUserName(), nodeAttrs.GetGroupName(), node .IsSymlink() ? node.AsSymlink().GetSymlink() : null, path, node.GetId(), childrenNum , feInfo, storagePolicy)); }
// do not store locations of last block /// <summary> /// Serialize a /// <see cref="INodeFile"/> /// node /// </summary> /// <param name="node">The node to write</param> /// <param name="out"> /// The /// <see cref="System.IO.DataOutputStream"/> /// where the fields are written /// </param> /// <param name="writeBlock">Whether to write block information</param> /// <exception cref="System.IO.IOException"/> public static void WriteINodeFile(INodeFile file, DataOutput @out, bool writeUnderConstruction ) { WriteLocalName(file, @out); @out.WriteLong(file.GetId()); @out.WriteShort(file.GetFileReplication()); @out.WriteLong(file.GetModificationTime()); @out.WriteLong(file.GetAccessTime()); @out.WriteLong(file.GetPreferredBlockSize()); WriteBlocks(file.GetBlocks(), @out); SnapshotFSImageFormat.SaveFileDiffList(file, @out); if (writeUnderConstruction) { if (file.IsUnderConstruction()) { @out.WriteBoolean(true); FileUnderConstructionFeature uc = file.GetFileUnderConstructionFeature(); WriteString(uc.GetClientName(), @out); WriteString(uc.GetClientMachine(), @out); } else { @out.WriteBoolean(false); } } WritePermissionStatus(file, @out); }
/// <summary>Create FileStatus with location info by file INode</summary> /// <exception cref="System.IO.IOException"/> private static HdfsLocatedFileStatus CreateLocatedFileStatus(FSDirectory fsd, string fullPath, byte[] path, INode node, byte storagePolicy, int snapshot, bool isRawPath , INodesInPath iip) { System.Diagnostics.Debug.Assert(fsd.HasReadLock()); long size = 0; // length is zero for directories short replication = 0; long blocksize = 0; LocatedBlocks loc = null; bool isEncrypted; FileEncryptionInfo feInfo = isRawPath ? null : fsd.GetFileEncryptionInfo(node, snapshot , iip); if (node.IsFile()) { INodeFile fileNode = node.AsFile(); size = fileNode.ComputeFileSize(snapshot); replication = fileNode.GetFileReplication(snapshot); blocksize = fileNode.GetPreferredBlockSize(); bool inSnapshot = snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .CurrentStateId; bool isUc = !inSnapshot && fileNode.IsUnderConstruction(); long fileSize = !inSnapshot && isUc?fileNode.ComputeFileSizeNotIncludingLastUcBlock () : size; loc = fsd.GetFSNamesystem().GetBlockManager().CreateLocatedBlocks(fileNode.GetBlocks (snapshot), fileSize, isUc, 0L, size, false, inSnapshot, feInfo); if (loc == null) { loc = new LocatedBlocks(); } isEncrypted = (feInfo != null) || (isRawPath && fsd.IsInAnEZ(INodesInPath.FromINode (node))); } else { isEncrypted = fsd.IsInAnEZ(INodesInPath.FromINode(node)); } int childrenNum = node.IsDirectory() ? node.AsDirectory().GetChildrenNum(snapshot ) : 0; INodeAttributes nodeAttrs = fsd.GetAttributes(fullPath, path, node, snapshot); HdfsLocatedFileStatus status = new HdfsLocatedFileStatus(size, node.IsDirectory() , replication, blocksize, node.GetModificationTime(snapshot), node.GetAccessTime (snapshot), GetPermissionForFileStatus(nodeAttrs, isEncrypted), nodeAttrs.GetUserName (), nodeAttrs.GetGroupName(), node.IsSymlink() ? node.AsSymlink().GetSymlink() : null, path, node.GetId(), loc, childrenNum, feInfo, storagePolicy); // Set caching information for the located blocks. if (loc != null) { CacheManager cacheManager = fsd.GetFSNamesystem().GetCacheManager(); foreach (LocatedBlock lb in loc.GetLocatedBlocks()) { cacheManager.SetCachedLocations(lb); } } return(status); }
// Helper function that writes an INodeUnderConstruction // into the output stream // /// <exception cref="System.IO.IOException"/> internal static void WriteINodeUnderConstruction(DataOutputStream @out, INodeFile cons, string path) { WriteString(path, @out); @out.WriteLong(cons.GetId()); @out.WriteShort(cons.GetFileReplication()); @out.WriteLong(cons.GetModificationTime()); @out.WriteLong(cons.GetPreferredBlockSize()); WriteBlocks(cons.GetBlocks(), @out); cons.GetPermissionStatus().Write(@out); FileUnderConstructionFeature uc = cons.GetFileUnderConstructionFeature(); WriteString(uc.GetClientName(), @out); WriteString(uc.GetClientMachine(), @out); @out.WriteInt(0); }
/// <exception cref="System.IO.IOException"/> private static INodeFile[] VerifySrcFiles(FSDirectory fsd, string[] srcs, INodesInPath targetIIP, FSPermissionChecker pc) { // to make sure no two files are the same ICollection <INodeFile> si = new LinkedHashSet <INodeFile>(); INodeFile targetINode = targetIIP.GetLastINode().AsFile(); INodeDirectory targetParent = targetINode.GetParent(); // now check the srcs foreach (string src in srcs) { INodesInPath iip = fsd.GetINodesInPath4Write(src); // permission check for srcs if (pc != null) { fsd.CheckPathAccess(pc, iip, FsAction.Read); // read the file fsd.CheckParentAccess(pc, iip, FsAction.Write); } // for delete INode srcINode = iip.GetLastINode(); INodeFile srcINodeFile = INodeFile.ValueOf(srcINode, src); // make sure the src file and the target file are in the same dir if (srcINodeFile.GetParent() != targetParent) { throw new HadoopIllegalArgumentException("Source file " + src + " is not in the same directory with the target " + targetIIP.GetPath()); } // make sure all the source files are not in snapshot if (srcINode.IsInLatestSnapshot(iip.GetLatestSnapshotId())) { throw new SnapshotException("Concat: the source file " + src + " is in snapshot"); } // check if the file has other references. if (srcINode.IsReference() && ((INodeReference.WithCount)srcINode.AsReference().GetReferredINode ()).GetReferenceCount() > 1) { throw new SnapshotException("Concat: the source file " + src + " is referred by some other reference in some snapshot." ); } // source file cannot be the same with the target file if (srcINode == targetINode) { throw new HadoopIllegalArgumentException("concat: the src file " + src + " is the same with the target file " + targetIIP.GetPath()); } // source file cannot be under construction or empty if (srcINodeFile.IsUnderConstruction() || srcINodeFile.NumBlocks() == 0) { throw new HadoopIllegalArgumentException("concat: source file " + src + " is invalid or empty or underConstruction" ); } // source file's preferred block size cannot be greater than the target // file if (srcINodeFile.GetPreferredBlockSize() > targetINode.GetPreferredBlockSize()) { throw new HadoopIllegalArgumentException("concat: source file " + src + " has preferred block size " + srcINodeFile.GetPreferredBlockSize() + " which is greater than the target file's preferred block size " + targetINode.GetPreferredBlockSize()); } si.AddItem(srcINodeFile); } // make sure no two files are the same if (si.Count < srcs.Length) { // it means at least two files are the same throw new HadoopIllegalArgumentException("concat: at least two of the source files are the same" ); } return(Sharpen.Collections.ToArray(si, new INodeFile[si.Count])); }