예제 #1
0
 public virtual void TestVerifyChunkedSumsSuccess()
 {
     AllocateDirectByteBuffers();
     FillDataAndValidChecksums();
     NativeCrc32.VerifyChunkedSums(bytesPerChecksum, checksumType.id, checksums, data,
                                   fileName, BasePosition);
 }
예제 #2
0
        /// <summary>Implementation of chunked calculation specifically on byte arrays.</summary>
        /// <remarks>
        /// Implementation of chunked calculation specifically on byte arrays. This
        /// is to avoid the copy when dealing with ByteBuffers that have array backing.
        /// </remarks>
        public virtual void CalculateChunkedSums(byte[] data, int dataOffset, int dataLength
                                                 , byte[] sums, int sumsOffset)
        {
            if (type.size == 0)
            {
                return;
            }
            if (NativeCrc32.IsAvailable())
            {
                NativeCrc32.CalculateChunkedSumsByteArray(bytesPerChecksum, type.id, sums, sumsOffset
                                                          , data, dataOffset, dataLength);
                return;
            }
            int remaining = dataLength;

            while (remaining > 0)
            {
                int n = Math.Min(remaining, bytesPerChecksum);
                summer.Reset();
                summer.Update(data, dataOffset, n);
                dataOffset += n;
                remaining  -= n;
                long calculated = summer.GetValue();
                sums[sumsOffset++] = unchecked ((byte)(calculated >> 24));
                sums[sumsOffset++] = unchecked ((byte)(calculated >> 16));
                sums[sumsOffset++] = unchecked ((byte)(calculated >> 8));
                sums[sumsOffset++] = unchecked ((byte)(calculated));
            }
        }
예제 #3
0
 public virtual void TestCalculateChunkedSumsFail()
 {
     AllocateDirectByteBuffers();
     FillDataAndInvalidChecksums();
     NativeCrc32.CalculateChunkedSums(bytesPerChecksum, checksumType.id, checksums, data
                                      );
 }
예제 #4
0
 public virtual void TestVerifyChunkedSumsByteArraySuccess()
 {
     AllocateArrayByteBuffers();
     FillDataAndValidChecksums();
     NativeCrc32.VerifyChunkedSumsByteArray(bytesPerChecksum, checksumType.id, ((byte[]
                                                                                 )checksums.Array()), checksums.Position(), ((byte[])data.Array()), data.Position
                                                (), data.Remaining(), fileName, BasePosition);
 }
예제 #5
0
 public virtual void TestVerifyChunkedSumsFail()
 {
     AllocateDirectByteBuffers();
     FillDataAndInvalidChecksums();
     exception.Expect(typeof(ChecksumException));
     NativeCrc32.VerifyChunkedSums(bytesPerChecksum, checksumType.id, checksums, data,
                                   fileName, BasePosition);
 }
예제 #6
0
 public virtual void TestCalculateChunkedSumsByteArrayFail()
 {
     AllocateArrayByteBuffers();
     FillDataAndInvalidChecksums();
     NativeCrc32.CalculateChunkedSumsByteArray(bytesPerChecksum, checksumType.id, ((byte
                                                                                    [])checksums.Array()), checksums.Position(), ((byte[])data.Array()), data.Position
                                                   (), data.Remaining());
 }
예제 #7
0
 public virtual void TestVerifyChunkedSumsByteArrayFail()
 {
     AllocateArrayByteBuffers();
     FillDataAndInvalidChecksums();
     exception.Expect(typeof(ChecksumException));
     NativeCrc32.VerifyChunkedSumsByteArray(bytesPerChecksum, checksumType.id, ((byte[]
                                                                                 )checksums.Array()), checksums.Position(), ((byte[])data.Array()), data.Position
                                                (), data.Remaining(), fileName, BasePosition);
 }
예제 #8
0
        public virtual void Setup()
        {
            Assume.AssumeTrue(NativeCrc32.IsAvailable());
            Assert.Equal("These tests assume they can write a checksum value as a 4-byte int."
                         , 4, checksumType.size);
            Configuration conf = new Configuration();

            bytesPerChecksum = conf.GetInt(IoBytesPerChecksumKey, IoBytesPerChecksumDefault);
            fileName         = this.GetType().Name;
            checksum         = DataChecksum.NewDataChecksum(checksumType, bytesPerChecksum);
        }
예제 #9
0
        /// <summary>Verify that the given checksums match the given data.</summary>
        /// <remarks>
        /// Verify that the given checksums match the given data.
        /// The 'mark' of the ByteBuffer parameters may be modified by this function,.
        /// but the position is maintained.
        /// </remarks>
        /// <param name="data">the DirectByteBuffer pointing to the data to verify.</param>
        /// <param name="checksums">
        /// the DirectByteBuffer pointing to a series of stored
        /// checksums
        /// </param>
        /// <param name="fileName">the name of the file being read, for error-reporting</param>
        /// <param name="basePos">the file position to which the start of 'data' corresponds</param>
        /// <exception cref="Org.Apache.Hadoop.FS.ChecksumException">if the checksums do not match
        ///     </exception>
        public virtual void VerifyChunkedSums(ByteBuffer data, ByteBuffer checksums, string
                                              fileName, long basePos)
        {
            if (type.size == 0)
            {
                return;
            }
            if (data.HasArray() && checksums.HasArray())
            {
                VerifyChunkedSums(((byte[])data.Array()), data.ArrayOffset() + data.Position(), data
                                  .Remaining(), ((byte[])checksums.Array()), checksums.ArrayOffset() + checksums.Position
                                      (), fileName, basePos);
                return;
            }
            if (NativeCrc32.IsAvailable())
            {
                NativeCrc32.VerifyChunkedSums(bytesPerChecksum, type.id, checksums, data, fileName
                                              , basePos);
                return;
            }
            int startDataPos = data.Position();

            data.Mark();
            checksums.Mark();
            try
            {
                byte[] buf = new byte[bytesPerChecksum];
                byte[] sum = new byte[type.size];
                while (data.Remaining() > 0)
                {
                    int n = Math.Min(data.Remaining(), bytesPerChecksum);
                    checksums.Get(sum);
                    data.Get(buf, 0, n);
                    summer.Reset();
                    summer.Update(buf, 0, n);
                    int calculated = (int)summer.GetValue();
                    int stored     = (sum[0] << 24 & unchecked ((int)(0xff000000))) | (sum[1] << 16 & unchecked (
                                                                                           (int)(0xff0000))) | (sum[2] << 8 & unchecked ((int)(0xff00))) | sum[3] & unchecked (
                        (int)(0xff));
                    if (calculated != stored)
                    {
                        long errPos = basePos + data.Position() - startDataPos - n;
                        throw new ChecksumException("Checksum error: " + fileName + " at " + errPos + " exp: "
                                                    + stored + " got: " + calculated, errPos);
                    }
                }
            }
            finally
            {
                data.Reset();
                checksums.Reset();
            }
        }
예제 #10
0
 /// <summary>Calculate checksums for the given data.</summary>
 /// <remarks>
 /// Calculate checksums for the given data.
 /// The 'mark' of the ByteBuffer parameters may be modified by this function,
 /// but the position is maintained.
 /// </remarks>
 /// <param name="data">the DirectByteBuffer pointing to the data to checksum.</param>
 /// <param name="checksums">
 /// the DirectByteBuffer into which checksums will be
 /// stored. Enough space must be available in this
 /// buffer to put the checksums.
 /// </param>
 public virtual void CalculateChunkedSums(ByteBuffer data, ByteBuffer checksums)
 {
     if (type.size == 0)
     {
         return;
     }
     if (data.HasArray() && checksums.HasArray())
     {
         CalculateChunkedSums(((byte[])data.Array()), data.ArrayOffset() + data.Position()
                              , data.Remaining(), ((byte[])checksums.Array()), checksums.ArrayOffset() + checksums
                              .Position());
         return;
     }
     if (NativeCrc32.IsAvailable())
     {
         NativeCrc32.CalculateChunkedSums(bytesPerChecksum, type.id, checksums, data);
         return;
     }
     data.Mark();
     checksums.Mark();
     try
     {
         byte[] buf = new byte[bytesPerChecksum];
         while (data.Remaining() > 0)
         {
             int n = Math.Min(data.Remaining(), bytesPerChecksum);
             data.Get(buf, 0, n);
             summer.Reset();
             summer.Update(buf, 0, n);
             checksums.PutInt((int)summer.GetValue());
         }
     }
     finally
     {
         data.Reset();
         checksums.Reset();
     }
 }
예제 #11
0
        /// <summary>Implementation of chunked verification specifically on byte arrays.</summary>
        /// <remarks>
        /// Implementation of chunked verification specifically on byte arrays. This
        /// is to avoid the copy when dealing with ByteBuffers that have array backing.
        /// </remarks>
        /// <exception cref="Org.Apache.Hadoop.FS.ChecksumException"/>
        private void VerifyChunkedSums(byte[] data, int dataOff, int dataLen, byte[] checksums
                                       , int checksumsOff, string fileName, long basePos)
        {
            if (type.size == 0)
            {
                return;
            }
            if (NativeCrc32.IsAvailable())
            {
                NativeCrc32.VerifyChunkedSumsByteArray(bytesPerChecksum, type.id, checksums, checksumsOff
                                                       , data, dataOff, dataLen, fileName, basePos);
                return;
            }
            int remaining = dataLen;
            int dataPos   = 0;

            while (remaining > 0)
            {
                int n = Math.Min(remaining, bytesPerChecksum);
                summer.Reset();
                summer.Update(data, dataOff + dataPos, n);
                dataPos   += n;
                remaining -= n;
                int calculated = (int)summer.GetValue();
                int stored     = (checksums[checksumsOff] << 24 & unchecked ((int)(0xff000000))) | (checksums
                                                                                                    [checksumsOff + 1] << 16 & unchecked ((int)(0xff0000))) | (checksums[checksumsOff
                                                                                                                                                                         + 2] << 8 & unchecked ((int)(0xff00))) | checksums[checksumsOff + 3] & unchecked (
                    (int)(0xff));
                checksumsOff += 4;
                if (calculated != stored)
                {
                    long errPos = basePos + dataPos - n;
                    throw new ChecksumException("Checksum error: " + fileName + " at " + errPos + " exp: "
                                                + stored + " got: " + calculated, errPos);
                }
            }
        }