/// <summary>Implementation of chunked calculation specifically on byte arrays.</summary>
/// <remarks>
/// Implementation of chunked calculation specifically on byte arrays. This
/// is to avoid the copy when dealing with ByteBuffers that have array backing.
/// </remarks>
public virtual void CalculateChunkedSums(byte[] data, int dataOffset, int dataLength
, byte[] sums, int sumsOffset)
{
if (type.size == 0)
{
return;
}
if (NativeCrc32.IsAvailable())
{
NativeCrc32.CalculateChunkedSumsByteArray(bytesPerChecksum, type.id, sums, sumsOffset
, data, dataOffset, dataLength);
return;
}
int remaining = dataLength;
while (remaining > 0)
{
int n = Math.Min(remaining, bytesPerChecksum);
summer.Reset();
summer.Update(data, dataOffset, n);
dataOffset += n;
remaining -= n;
long calculated = summer.GetValue();
sums[sumsOffset++] = unchecked ((byte)(calculated >> 24));
sums[sumsOffset++] = unchecked ((byte)(calculated >> 16));
sums[sumsOffset++] = unchecked ((byte)(calculated >> 8));
sums[sumsOffset++] = unchecked ((byte)(calculated));
}
}
public virtual void Setup()
{
Assume.AssumeTrue(NativeCrc32.IsAvailable());
Assert.Equal("These tests assume they can write a checksum value as a 4-byte int."
, 4, checksumType.size);
Configuration conf = new Configuration();
bytesPerChecksum = conf.GetInt(IoBytesPerChecksumKey, IoBytesPerChecksumDefault);
fileName = this.GetType().Name;
checksum = DataChecksum.NewDataChecksum(checksumType, bytesPerChecksum);
}
/// <summary>Verify that the given checksums match the given data.</summary>
/// <remarks>
/// Verify that the given checksums match the given data.
/// The 'mark' of the ByteBuffer parameters may be modified by this function,.
/// but the position is maintained.
/// </remarks>
/// <param name="data">the DirectByteBuffer pointing to the data to verify.</param>
/// <param name="checksums">
/// the DirectByteBuffer pointing to a series of stored
/// checksums
/// </param>
/// <param name="fileName">the name of the file being read, for error-reporting</param>
/// <param name="basePos">the file position to which the start of 'data' corresponds</param>
/// <exception cref="Org.Apache.Hadoop.FS.ChecksumException">if the checksums do not match
/// </exception>
public virtual void VerifyChunkedSums(ByteBuffer data, ByteBuffer checksums, string
fileName, long basePos)
{
if (type.size == 0)
{
return;
}
if (data.HasArray() && checksums.HasArray())
{
VerifyChunkedSums(((byte[])data.Array()), data.ArrayOffset() + data.Position(), data
.Remaining(), ((byte[])checksums.Array()), checksums.ArrayOffset() + checksums.Position
(), fileName, basePos);
return;
}
if (NativeCrc32.IsAvailable())
{
NativeCrc32.VerifyChunkedSums(bytesPerChecksum, type.id, checksums, data, fileName
, basePos);
return;
}
int startDataPos = data.Position();
data.Mark();
checksums.Mark();
try
{
byte[] buf = new byte[bytesPerChecksum];
byte[] sum = new byte[type.size];
while (data.Remaining() > 0)
{
int n = Math.Min(data.Remaining(), bytesPerChecksum);
checksums.Get(sum);
data.Get(buf, 0, n);
summer.Reset();
summer.Update(buf, 0, n);
int calculated = (int)summer.GetValue();
int stored = (sum[0] << 24 & unchecked ((int)(0xff000000))) | (sum[1] << 16 & unchecked (
(int)(0xff0000))) | (sum[2] << 8 & unchecked ((int)(0xff00))) | sum[3] & unchecked (
(int)(0xff));
if (calculated != stored)
{
long errPos = basePos + data.Position() - startDataPos - n;
throw new ChecksumException("Checksum error: " + fileName + " at " + errPos + " exp: "
+ stored + " got: " + calculated, errPos);
}
}
}
finally
{
data.Reset();
checksums.Reset();
}
}
/// <summary>Calculate checksums for the given data.</summary>
/// <remarks>
/// Calculate checksums for the given data.
/// The 'mark' of the ByteBuffer parameters may be modified by this function,
/// but the position is maintained.
/// </remarks>
/// <param name="data">the DirectByteBuffer pointing to the data to checksum.</param>
/// <param name="checksums">
/// the DirectByteBuffer into which checksums will be
/// stored. Enough space must be available in this
/// buffer to put the checksums.
/// </param>
public virtual void CalculateChunkedSums(ByteBuffer data, ByteBuffer checksums)
{
if (type.size == 0)
{
return;
}
if (data.HasArray() && checksums.HasArray())
{
CalculateChunkedSums(((byte[])data.Array()), data.ArrayOffset() + data.Position()
, data.Remaining(), ((byte[])checksums.Array()), checksums.ArrayOffset() + checksums
.Position());
return;
}
if (NativeCrc32.IsAvailable())
{
NativeCrc32.CalculateChunkedSums(bytesPerChecksum, type.id, checksums, data);
return;
}
data.Mark();
checksums.Mark();
try
{
byte[] buf = new byte[bytesPerChecksum];
while (data.Remaining() > 0)
{
int n = Math.Min(data.Remaining(), bytesPerChecksum);
data.Get(buf, 0, n);
summer.Reset();
summer.Update(buf, 0, n);
checksums.PutInt((int)summer.GetValue());
}
}
finally
{
data.Reset();
checksums.Reset();
}
}
/// <summary>Implementation of chunked verification specifically on byte arrays.</summary>
/// <remarks>
/// Implementation of chunked verification specifically on byte arrays. This
/// is to avoid the copy when dealing with ByteBuffers that have array backing.
/// </remarks>
/// <exception cref="Org.Apache.Hadoop.FS.ChecksumException"/>
private void VerifyChunkedSums(byte[] data, int dataOff, int dataLen, byte[] checksums
, int checksumsOff, string fileName, long basePos)
{
if (type.size == 0)
{
return;
}
if (NativeCrc32.IsAvailable())
{
NativeCrc32.VerifyChunkedSumsByteArray(bytesPerChecksum, type.id, checksums, checksumsOff
, data, dataOff, dataLen, fileName, basePos);
return;
}
int remaining = dataLen;
int dataPos = 0;
while (remaining > 0)
{
int n = Math.Min(remaining, bytesPerChecksum);
summer.Reset();
summer.Update(data, dataOff + dataPos, n);
dataPos += n;
remaining -= n;
int calculated = (int)summer.GetValue();
int stored = (checksums[checksumsOff] << 24 & unchecked ((int)(0xff000000))) | (checksums
[checksumsOff + 1] << 16 & unchecked ((int)(0xff0000))) | (checksums[checksumsOff
+ 2] << 8 & unchecked ((int)(0xff00))) | checksums[checksumsOff + 3] & unchecked (
(int)(0xff));
checksumsOff += 4;
if (calculated != stored)
{
long errPos = basePos + dataPos - n;
throw new ChecksumException("Checksum error: " + fileName + " at " + errPos + " exp: "
+ stored + " got: " + calculated, errPos);
}
}
}