/// <exception cref="System.IO.IOException"/> public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> flag , FsPermission absolutePermission, int bufferSize, short replication, long blockSize , Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent) { // call to primitiveCreate CheckPath(f); // Default impl assumes that permissions do not matter // calling the regular create is good enough. // FSs that implement permissions should override this. if (!createParent) { // parent must exist. // since this.create makes parent dirs automatically // we must throw exception if parent does not exist. FileStatus stat = GetFileStatus(f.GetParent()); if (stat == null) { throw new FileNotFoundException("Missing parent:" + f); } if (!stat.IsDirectory()) { throw new ParentNotDirectoryException("parent is not a dir:" + f); } } // parent does exist - go ahead with create of file. return(fsImpl.PrimitiveCreate(f, absolutePermission, flag, bufferSize, replication , blockSize, progress, checksumOpt)); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/> public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> flag , FsPermission absolutePermission, int bufferSize, short replication, long blockSize , Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent) { CheckPath(f); return(myFs.CreateInternal(f, flag, absolutePermission, bufferSize, replication, blockSize, progress, checksumOpt, createParent)); }
private void CheckParams(Options.ChecksumOpt expected, Options.ChecksumOpt obtained ) { Assert.Equal(expected.GetChecksumType(), obtained.GetChecksumType ()); Assert.Equal(expected.GetBytesPerChecksum(), obtained.GetBytesPerChecksum ()); }
/// <exception cref="System.IO.IOException"/> public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> createFlag , FsPermission absolutePermission, int bufferSize, short replication, long blockSize , Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent) { DFSOutputStream dfsos = dfs.PrimitiveCreate(GetUriPath(f), absolutePermission, createFlag , createParent, replication, blockSize, progress, bufferSize, checksumOpt); return(dfs.CreateWrappedOutputStream(dfsos, statistics, dfsos.GetInitialLen())); }
/// <exception cref="System.IO.IOException"/> public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> createFlag , FsPermission absolutePermission, int bufferSize, short replication, long blockSize , Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent) { FSDataOutputStream @out = new FSDataOutputStream(new ChecksumFs.ChecksumFSOutputSummer (this, f, createFlag, absolutePermission, bufferSize, replication, blockSize, progress , checksumOpt, createParent), null); return(@out); }
/// <exception cref="System.IO.IOException"/> public ChecksumFSOutputSummer(ChecksumFs fs, Path file, EnumSet <CreateFlag> createFlag , FsPermission absolutePermission, int bufferSize, short replication, long blockSize , Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent) : base(DataChecksum.NewDataChecksum(DataChecksum.Type.Crc32, fs.GetBytesPerSum()) ) { // checksumOpt is passed down to the raw fs. Unless it implements // checksum impelemts internally, checksumOpt will be ignored. // If the raw fs does checksum internally, we will end up with // two layers of checksumming. i.e. checksumming checksum file. this.datas = fs.GetRawFs().CreateInternal(file, createFlag, absolutePermission, bufferSize , replication, blockSize, progress, checksumOpt, createParent); // Now create the chekcsumfile; adjust the buffsize int bytesPerSum = fs.GetBytesPerSum(); int sumBufferSize = fs.GetSumBufferSize(bytesPerSum, bufferSize); this.sums = fs.GetRawFs().CreateInternal(fs.GetChecksumFile(file), EnumSet.Of(CreateFlag .Create, CreateFlag.Overwrite), absolutePermission, sumBufferSize, replication, blockSize, progress, checksumOpt, createParent); sums.Write(ChecksumVersion, 0, ChecksumVersion.Length); sums.WriteInt(bytesPerSum); }
/// <summary> /// A helper method for processing user input and default value to /// create a combined checksum option. /// </summary> /// <remarks> /// A helper method for processing user input and default value to /// create a combined checksum option. This is a bit complicated because /// bytesPerChecksum is kept for backward compatibility. /// </remarks> /// <param name="defaultOpt">Default checksum option</param> /// <param name="userOpt">User-specified checksum option. Ignored if null.</param> /// <param name="userBytesPerChecksum"> /// User-specified bytesPerChecksum /// Ignored if < 0. /// </param> public static Options.ChecksumOpt ProcessChecksumOpt(Options.ChecksumOpt defaultOpt , Options.ChecksumOpt userOpt, int userBytesPerChecksum) { bool useDefaultType; DataChecksum.Type type; if (userOpt != null && userOpt.GetChecksumType() != DataChecksum.Type.Default) { useDefaultType = false; type = userOpt.GetChecksumType(); } else { useDefaultType = true; type = defaultOpt.GetChecksumType(); } // bytesPerChecksum - order of preference // user specified value in bytesPerChecksum // user specified value in checksumOpt // default. if (userBytesPerChecksum > 0) { return(new Options.ChecksumOpt(type, userBytesPerChecksum)); } else { if (userOpt != null && userOpt.GetBytesPerChecksum() > 0) { return(!useDefaultType ? userOpt : new Options.ChecksumOpt(type, userOpt.GetBytesPerChecksum ())); } else { return(useDefaultType ? defaultOpt : new Options.ChecksumOpt(type, defaultOpt.GetBytesPerChecksum ())); } } }
public virtual void TestProcessChecksumOpt() { Options.ChecksumOpt defaultOpt = new Options.ChecksumOpt(DataChecksum.Type.Crc32, 512); Options.ChecksumOpt finalOpt; // Give a null finalOpt = Options.ChecksumOpt.ProcessChecksumOpt(defaultOpt, null); CheckParams(defaultOpt, finalOpt); // null with bpc finalOpt = Options.ChecksumOpt.ProcessChecksumOpt(defaultOpt, null, 1024); CheckParams(DataChecksum.Type.Crc32, 1024, finalOpt); Options.ChecksumOpt myOpt = new Options.ChecksumOpt(); // custom with unspecified parameters finalOpt = Options.ChecksumOpt.ProcessChecksumOpt(defaultOpt, myOpt); CheckParams(defaultOpt, finalOpt); myOpt = new Options.ChecksumOpt(DataChecksum.Type.Crc32c, 2048); // custom config finalOpt = Options.ChecksumOpt.ProcessChecksumOpt(defaultOpt, myOpt); CheckParams(DataChecksum.Type.Crc32c, 2048, finalOpt); // custom config + bpc finalOpt = Options.ChecksumOpt.ProcessChecksumOpt(defaultOpt, myOpt, 4096); CheckParams(DataChecksum.Type.Crc32c, 4096, finalOpt); }
/// <summary> /// A helper method for processing user input and default value to /// create a combined checksum option. /// </summary> /// <param name="defaultOpt">Default checksum option</param> /// <param name="userOpt">User-specified checksum option</param> public static Options.ChecksumOpt ProcessChecksumOpt(Options.ChecksumOpt defaultOpt , Options.ChecksumOpt userOpt) { return(ProcessChecksumOpt(defaultOpt, userOpt, -1)); }
public static Options.CreateOpts.ChecksumParam ChecksumParam(Options.ChecksumOpt csumOpt) { return(new Options.CreateOpts.ChecksumParam(csumOpt)); }
protected internal ChecksumParam(Options.ChecksumOpt csumOpt) { checksumOpt = csumOpt; }
/// <exception cref="System.IO.IOException"/> public virtual FSDataOutputStream Create(Path f, FsPermission permission, EnumSet <CreateFlag> flags, int bufferSize, short replication, long blockSize, Progressable progress, Options.ChecksumOpt checksumOpt) { return(null); }
private void CheckParams(DataChecksum.Type type, int bpc, Options.ChecksumOpt obtained ) { Assert.Equal(type, obtained.GetChecksumType()); Assert.Equal(bpc, obtained.GetBytesPerChecksum()); }
/// <exception cref="System.IO.IOException"/> public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> flag , FsPermission absolutePermission, int bufferSize, short replication, long blockSize , Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent) { // deliberately empty return(null); }