/// <exception cref="System.IO.IOException"/>
public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> flag
, FsPermission absolutePermission, int bufferSize, short replication, long blockSize
, Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent)
{
// call to primitiveCreate
CheckPath(f);
// Default impl assumes that permissions do not matter
// calling the regular create is good enough.
// FSs that implement permissions should override this.
if (!createParent)
{
// parent must exist.
// since this.create makes parent dirs automatically
// we must throw exception if parent does not exist.
FileStatus stat = GetFileStatus(f.GetParent());
if (stat == null)
{
throw new FileNotFoundException("Missing parent:" + f);
}
if (!stat.IsDirectory())
{
throw new ParentNotDirectoryException("parent is not a dir:" + f);
}
}
// parent does exist - go ahead with create of file.
return(fsImpl.PrimitiveCreate(f, absolutePermission, flag, bufferSize, replication
, blockSize, progress, checksumOpt));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/>
public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> flag
, FsPermission absolutePermission, int bufferSize, short replication, long blockSize
, Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent)
{
CheckPath(f);
return(myFs.CreateInternal(f, flag, absolutePermission, bufferSize, replication,
blockSize, progress, checksumOpt, createParent));
}
private void CheckParams(Options.ChecksumOpt expected, Options.ChecksumOpt obtained
)
{
Assert.Equal(expected.GetChecksumType(), obtained.GetChecksumType
());
Assert.Equal(expected.GetBytesPerChecksum(), obtained.GetBytesPerChecksum
());
}
/// <exception cref="System.IO.IOException"/>
public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> createFlag
, FsPermission absolutePermission, int bufferSize, short replication, long blockSize
, Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent)
{
DFSOutputStream dfsos = dfs.PrimitiveCreate(GetUriPath(f), absolutePermission, createFlag
, createParent, replication, blockSize, progress, bufferSize, checksumOpt);
return(dfs.CreateWrappedOutputStream(dfsos, statistics, dfsos.GetInitialLen()));
}
/// <exception cref="System.IO.IOException"/>
public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> createFlag
, FsPermission absolutePermission, int bufferSize, short replication, long blockSize
, Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent)
{
FSDataOutputStream @out = new FSDataOutputStream(new ChecksumFs.ChecksumFSOutputSummer
(this, f, createFlag, absolutePermission, bufferSize, replication, blockSize, progress
, checksumOpt, createParent), null);
return(@out);
}
/// <exception cref="System.IO.IOException"/>
public ChecksumFSOutputSummer(ChecksumFs fs, Path file, EnumSet <CreateFlag> createFlag
, FsPermission absolutePermission, int bufferSize, short replication, long blockSize
, Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent)
: base(DataChecksum.NewDataChecksum(DataChecksum.Type.Crc32, fs.GetBytesPerSum())
)
{
// checksumOpt is passed down to the raw fs. Unless it implements
// checksum impelemts internally, checksumOpt will be ignored.
// If the raw fs does checksum internally, we will end up with
// two layers of checksumming. i.e. checksumming checksum file.
this.datas = fs.GetRawFs().CreateInternal(file, createFlag, absolutePermission, bufferSize
, replication, blockSize, progress, checksumOpt, createParent);
// Now create the chekcsumfile; adjust the buffsize
int bytesPerSum = fs.GetBytesPerSum();
int sumBufferSize = fs.GetSumBufferSize(bytesPerSum, bufferSize);
this.sums = fs.GetRawFs().CreateInternal(fs.GetChecksumFile(file), EnumSet.Of(CreateFlag
.Create, CreateFlag.Overwrite), absolutePermission, sumBufferSize, replication,
blockSize, progress, checksumOpt, createParent);
sums.Write(ChecksumVersion, 0, ChecksumVersion.Length);
sums.WriteInt(bytesPerSum);
}
/// <summary>
/// A helper method for processing user input and default value to
/// create a combined checksum option.
/// </summary>
/// <remarks>
/// A helper method for processing user input and default value to
/// create a combined checksum option. This is a bit complicated because
/// bytesPerChecksum is kept for backward compatibility.
/// </remarks>
/// <param name="defaultOpt">Default checksum option</param>
/// <param name="userOpt">User-specified checksum option. Ignored if null.</param>
/// <param name="userBytesPerChecksum">
/// User-specified bytesPerChecksum
/// Ignored if < 0.
/// </param>
public static Options.ChecksumOpt ProcessChecksumOpt(Options.ChecksumOpt defaultOpt
, Options.ChecksumOpt userOpt, int userBytesPerChecksum)
{
bool useDefaultType;
DataChecksum.Type type;
if (userOpt != null && userOpt.GetChecksumType() != DataChecksum.Type.Default)
{
useDefaultType = false;
type = userOpt.GetChecksumType();
}
else
{
useDefaultType = true;
type = defaultOpt.GetChecksumType();
}
// bytesPerChecksum - order of preference
// user specified value in bytesPerChecksum
// user specified value in checksumOpt
// default.
if (userBytesPerChecksum > 0)
{
return(new Options.ChecksumOpt(type, userBytesPerChecksum));
}
else
{
if (userOpt != null && userOpt.GetBytesPerChecksum() > 0)
{
return(!useDefaultType ? userOpt : new Options.ChecksumOpt(type, userOpt.GetBytesPerChecksum
()));
}
else
{
return(useDefaultType ? defaultOpt : new Options.ChecksumOpt(type, defaultOpt.GetBytesPerChecksum
()));
}
}
}
public virtual void TestProcessChecksumOpt()
{
Options.ChecksumOpt defaultOpt = new Options.ChecksumOpt(DataChecksum.Type.Crc32,
512);
Options.ChecksumOpt finalOpt;
// Give a null
finalOpt = Options.ChecksumOpt.ProcessChecksumOpt(defaultOpt, null);
CheckParams(defaultOpt, finalOpt);
// null with bpc
finalOpt = Options.ChecksumOpt.ProcessChecksumOpt(defaultOpt, null, 1024);
CheckParams(DataChecksum.Type.Crc32, 1024, finalOpt);
Options.ChecksumOpt myOpt = new Options.ChecksumOpt();
// custom with unspecified parameters
finalOpt = Options.ChecksumOpt.ProcessChecksumOpt(defaultOpt, myOpt);
CheckParams(defaultOpt, finalOpt);
myOpt = new Options.ChecksumOpt(DataChecksum.Type.Crc32c, 2048);
// custom config
finalOpt = Options.ChecksumOpt.ProcessChecksumOpt(defaultOpt, myOpt);
CheckParams(DataChecksum.Type.Crc32c, 2048, finalOpt);
// custom config + bpc
finalOpt = Options.ChecksumOpt.ProcessChecksumOpt(defaultOpt, myOpt, 4096);
CheckParams(DataChecksum.Type.Crc32c, 4096, finalOpt);
}
/// <summary>
/// A helper method for processing user input and default value to
/// create a combined checksum option.
/// </summary>
/// <param name="defaultOpt">Default checksum option</param>
/// <param name="userOpt">User-specified checksum option</param>
public static Options.ChecksumOpt ProcessChecksumOpt(Options.ChecksumOpt defaultOpt
, Options.ChecksumOpt userOpt)
{
return(ProcessChecksumOpt(defaultOpt, userOpt, -1));
}
public static Options.CreateOpts.ChecksumParam ChecksumParam(Options.ChecksumOpt
csumOpt)
{
return(new Options.CreateOpts.ChecksumParam(csumOpt));
}
protected internal ChecksumParam(Options.ChecksumOpt csumOpt)
{
checksumOpt = csumOpt;
}
/// <exception cref="System.IO.IOException"/>
public virtual FSDataOutputStream Create(Path f, FsPermission permission, EnumSet
<CreateFlag> flags, int bufferSize, short replication, long blockSize, Progressable
progress, Options.ChecksumOpt checksumOpt)
{
return(null);
}
private void CheckParams(DataChecksum.Type type, int bpc, Options.ChecksumOpt obtained
)
{
Assert.Equal(type, obtained.GetChecksumType());
Assert.Equal(bpc, obtained.GetBytesPerChecksum());
}
/// <exception cref="System.IO.IOException"/>
public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> flag
, FsPermission absolutePermission, int bufferSize, short replication, long blockSize
, Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent)
{
// deliberately empty
return(null);
}
