Exemplo n.º 1
0
 /// <exception cref="System.IO.IOException"/>
 public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> flag
                                                   , FsPermission absolutePermission, int bufferSize, short replication, long blockSize
                                                   , Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent)
 {
     // call to primitiveCreate
     CheckPath(f);
     // Default impl assumes that permissions do not matter
     // calling the regular create is good enough.
     // FSs that implement permissions should override this.
     if (!createParent)
     {
         // parent must exist.
         // since this.create makes parent dirs automatically
         // we must throw exception if parent does not exist.
         FileStatus stat = GetFileStatus(f.GetParent());
         if (stat == null)
         {
             throw new FileNotFoundException("Missing parent:" + f);
         }
         if (!stat.IsDirectory())
         {
             throw new ParentNotDirectoryException("parent is not a dir:" + f);
         }
     }
     // parent does exist - go ahead with create of file.
     return(fsImpl.PrimitiveCreate(f, absolutePermission, flag, bufferSize, replication
                                   , blockSize, progress, checksumOpt));
 }
Exemplo n.º 2
0
 /// <exception cref="System.IO.IOException"/>
 /// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/>
 public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> flag
                                                   , FsPermission absolutePermission, int bufferSize, short replication, long blockSize
                                                   , Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent)
 {
     CheckPath(f);
     return(myFs.CreateInternal(f, flag, absolutePermission, bufferSize, replication,
                                blockSize, progress, checksumOpt, createParent));
 }
Exemplo n.º 3
0
 private void CheckParams(Options.ChecksumOpt expected, Options.ChecksumOpt obtained
                          )
 {
     Assert.Equal(expected.GetChecksumType(), obtained.GetChecksumType
                      ());
     Assert.Equal(expected.GetBytesPerChecksum(), obtained.GetBytesPerChecksum
                      ());
 }
Exemplo n.º 4
0
        /// <exception cref="System.IO.IOException"/>
        public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> createFlag
                                                          , FsPermission absolutePermission, int bufferSize, short replication, long blockSize
                                                          , Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent)
        {
            DFSOutputStream dfsos = dfs.PrimitiveCreate(GetUriPath(f), absolutePermission, createFlag
                                                        , createParent, replication, blockSize, progress, bufferSize, checksumOpt);

            return(dfs.CreateWrappedOutputStream(dfsos, statistics, dfsos.GetInitialLen()));
        }
Exemplo n.º 5
0
        /// <exception cref="System.IO.IOException"/>
        public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> createFlag
                                                          , FsPermission absolutePermission, int bufferSize, short replication, long blockSize
                                                          , Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent)
        {
            FSDataOutputStream @out = new FSDataOutputStream(new ChecksumFs.ChecksumFSOutputSummer
                                                                 (this, f, createFlag, absolutePermission, bufferSize, replication, blockSize, progress
                                                                 , checksumOpt, createParent), null);

            return(@out);
        }
Exemplo n.º 6
0
            /// <exception cref="System.IO.IOException"/>
            public ChecksumFSOutputSummer(ChecksumFs fs, Path file, EnumSet <CreateFlag> createFlag
                                          , FsPermission absolutePermission, int bufferSize, short replication, long blockSize
                                          , Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent)
                : base(DataChecksum.NewDataChecksum(DataChecksum.Type.Crc32, fs.GetBytesPerSum())
                       )
            {
                // checksumOpt is passed down to the raw fs. Unless it implements
                // checksum impelemts internally, checksumOpt will be ignored.
                // If the raw fs does checksum internally, we will end up with
                // two layers of checksumming. i.e. checksumming checksum file.
                this.datas = fs.GetRawFs().CreateInternal(file, createFlag, absolutePermission, bufferSize
                                                          , replication, blockSize, progress, checksumOpt, createParent);
                // Now create the chekcsumfile; adjust the buffsize
                int bytesPerSum   = fs.GetBytesPerSum();
                int sumBufferSize = fs.GetSumBufferSize(bytesPerSum, bufferSize);

                this.sums = fs.GetRawFs().CreateInternal(fs.GetChecksumFile(file), EnumSet.Of(CreateFlag
                                                                                              .Create, CreateFlag.Overwrite), absolutePermission, sumBufferSize, replication,
                                                         blockSize, progress, checksumOpt, createParent);
                sums.Write(ChecksumVersion, 0, ChecksumVersion.Length);
                sums.WriteInt(bytesPerSum);
            }
Exemplo n.º 7
0
            /// <summary>
            /// A helper method for processing user input and default value to
            /// create a combined checksum option.
            /// </summary>
            /// <remarks>
            /// A helper method for processing user input and default value to
            /// create a combined checksum option. This is a bit complicated because
            /// bytesPerChecksum is kept for backward compatibility.
            /// </remarks>
            /// <param name="defaultOpt">Default checksum option</param>
            /// <param name="userOpt">User-specified checksum option. Ignored if null.</param>
            /// <param name="userBytesPerChecksum">
            /// User-specified bytesPerChecksum
            /// Ignored if &lt; 0.
            /// </param>
            public static Options.ChecksumOpt ProcessChecksumOpt(Options.ChecksumOpt defaultOpt
                                                                 , Options.ChecksumOpt userOpt, int userBytesPerChecksum)
            {
                bool useDefaultType;

                DataChecksum.Type type;
                if (userOpt != null && userOpt.GetChecksumType() != DataChecksum.Type.Default)
                {
                    useDefaultType = false;
                    type           = userOpt.GetChecksumType();
                }
                else
                {
                    useDefaultType = true;
                    type           = defaultOpt.GetChecksumType();
                }
                //  bytesPerChecksum - order of preference
                //    user specified value in bytesPerChecksum
                //    user specified value in checksumOpt
                //    default.
                if (userBytesPerChecksum > 0)
                {
                    return(new Options.ChecksumOpt(type, userBytesPerChecksum));
                }
                else
                {
                    if (userOpt != null && userOpt.GetBytesPerChecksum() > 0)
                    {
                        return(!useDefaultType ? userOpt : new Options.ChecksumOpt(type, userOpt.GetBytesPerChecksum
                                                                                       ()));
                    }
                    else
                    {
                        return(useDefaultType ? defaultOpt : new Options.ChecksumOpt(type, defaultOpt.GetBytesPerChecksum
                                                                                         ()));
                    }
                }
            }
Exemplo n.º 8
0
 public virtual void TestProcessChecksumOpt()
 {
     Options.ChecksumOpt defaultOpt = new Options.ChecksumOpt(DataChecksum.Type.Crc32,
                                                              512);
     Options.ChecksumOpt finalOpt;
     // Give a null
     finalOpt = Options.ChecksumOpt.ProcessChecksumOpt(defaultOpt, null);
     CheckParams(defaultOpt, finalOpt);
     // null with bpc
     finalOpt = Options.ChecksumOpt.ProcessChecksumOpt(defaultOpt, null, 1024);
     CheckParams(DataChecksum.Type.Crc32, 1024, finalOpt);
     Options.ChecksumOpt myOpt = new Options.ChecksumOpt();
     // custom with unspecified parameters
     finalOpt = Options.ChecksumOpt.ProcessChecksumOpt(defaultOpt, myOpt);
     CheckParams(defaultOpt, finalOpt);
     myOpt = new Options.ChecksumOpt(DataChecksum.Type.Crc32c, 2048);
     // custom config
     finalOpt = Options.ChecksumOpt.ProcessChecksumOpt(defaultOpt, myOpt);
     CheckParams(DataChecksum.Type.Crc32c, 2048, finalOpt);
     // custom config + bpc
     finalOpt = Options.ChecksumOpt.ProcessChecksumOpt(defaultOpt, myOpt, 4096);
     CheckParams(DataChecksum.Type.Crc32c, 4096, finalOpt);
 }
Exemplo n.º 9
0
 /// <summary>
 /// A helper method for processing user input and default value to
 /// create a combined checksum option.
 /// </summary>
 /// <param name="defaultOpt">Default checksum option</param>
 /// <param name="userOpt">User-specified checksum option</param>
 public static Options.ChecksumOpt ProcessChecksumOpt(Options.ChecksumOpt defaultOpt
                                                      , Options.ChecksumOpt userOpt)
 {
     return(ProcessChecksumOpt(defaultOpt, userOpt, -1));
 }
Exemplo n.º 10
0
 public static Options.CreateOpts.ChecksumParam ChecksumParam(Options.ChecksumOpt
                                                              csumOpt)
 {
     return(new Options.CreateOpts.ChecksumParam(csumOpt));
 }
Exemplo n.º 11
0
 protected internal ChecksumParam(Options.ChecksumOpt csumOpt)
 {
     checksumOpt = csumOpt;
 }
Exemplo n.º 12
0
 /// <exception cref="System.IO.IOException"/>
 public virtual FSDataOutputStream Create(Path f, FsPermission permission, EnumSet
                                          <CreateFlag> flags, int bufferSize, short replication, long blockSize, Progressable
                                          progress, Options.ChecksumOpt checksumOpt)
 {
     return(null);
 }
Exemplo n.º 13
0
 private void CheckParams(DataChecksum.Type type, int bpc, Options.ChecksumOpt obtained
                          )
 {
     Assert.Equal(type, obtained.GetChecksumType());
     Assert.Equal(bpc, obtained.GetBytesPerChecksum());
 }
Exemplo n.º 14
0
 /// <exception cref="System.IO.IOException"/>
 public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> flag
                                                   , FsPermission absolutePermission, int bufferSize, short replication, long blockSize
                                                   , Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent)
 {
     // deliberately empty
     return(null);
 }