public void Done()
{
lock (_lock)
{
if (_done)
{
return;
}
_done = true;
_messageDigest.TransformFinalBlock(new byte[0], 0, 0);
_digest = _messageDigest.Hash;
_messageDigest.Dispose();
_messageDigest = null;
_dataSink = null;
}
}
private static IDictionary <ContentDigestAlgorithm, byte[]> ComputeContentDigests(
ISet <ContentDigestAlgorithm> digestAlgorithms,
Stream[] contents)
{
// For each digest algorithm the result is computed as follows:
// 1. Each segment of contents is split into consecutive chunks of 1 MB in size.
// The final chunk will be shorter iff the length of segment is not a multiple of 1 MB.
// No chunks are produced for empty (zero length) segments.
// 2. The digest of each chunk is computed over the concatenation of byte 0xa5, the chunk's
// length in bytes (uint32 little-endian) and the chunk's contents.
// 3. The output digest is computed over the concatenation of the byte 0x5a, the number of
// chunks (uint32 little-endian) and the concatenation of digests of chunks of all
// segments in-order.
long chunkCountLong = 0;
foreach (var input in contents)
{
chunkCountLong += GetChunkCount(input.Length, ContentDigestedChunkMaxSizeBytes);
}
if (chunkCountLong > int.MaxValue)
{
throw new CryptographicException("Input too long: " + chunkCountLong + " chunks");
}
var chunkCount = (int)chunkCountLong;
var digestAlgorithmsArray =
digestAlgorithms.ToArray();
var mds = new HashAlgorithm[digestAlgorithmsArray.Length];
var digestsOfChunks = new byte[digestAlgorithmsArray.Length][];
var digestOutputSizes = new int[digestAlgorithmsArray.Length];
for (var i = 0; i < digestAlgorithmsArray.Length; i++)
{
var digestAlgorithm = digestAlgorithmsArray[i];
var digestOutputSizeBytes = digestAlgorithm.ChunkDigestOutputSizeBytes;
digestOutputSizes[i] = digestOutputSizeBytes;
var concatenationOfChunkCountAndChunkDigests =
new byte[5 + chunkCount * digestOutputSizeBytes];
concatenationOfChunkCountAndChunkDigests[0] = 0x5a;
SetUnsignedInt32LittleEndian(
chunkCount, concatenationOfChunkCountAndChunkDigests, 1);
digestsOfChunks[i] = concatenationOfChunkCountAndChunkDigests;
var jcaAlgorithm = digestAlgorithm.JcaMessageDigestAlgorithm;
mds[i] = HashAlgorithm.Create(jcaAlgorithm);
}
var mdSink = new MessageDigestStream(mds);
var chunkContentPrefix = new byte[5];
chunkContentPrefix[0] = 0xa5;
var chunkIndex = 0;
// Optimization opportunity: digests of chunks can be computed in parallel. However,
// determining the number of computations to be performed in parallel is non-trivial. This
// depends on a wide range of factors, such as data source type (e.g., in-memory or fetched
// from file), CPU/memory/disk cache bandwidth and latency, interconnect architecture of CPU
// cores, load on the system from other threads of execution and other processes, size of
// input.
// For now, we compute these digests sequentially and thus have the luxury of improving
// performance by writing the digest of each chunk into a pre-allocated buffer at exactly
// the right position. This avoids unnecessary allocations, copying, and enables the final
// digest to be more efficient because it's presented with all of its input in one go.
foreach (var input in contents)
{
input.Position = 0;
long inputOffset = 0;
var inputRemaining = input.Length;
while (inputRemaining > 0)
{
var chunkSize =
(int)Math.Min(inputRemaining, ContentDigestedChunkMaxSizeBytes);
SetUnsignedInt32LittleEndian(chunkSize, chunkContentPrefix, 1);
foreach (var md in mds)
{
md.TransformBlock(chunkContentPrefix, 0, chunkContentPrefix.Length, chunkContentPrefix, 0);
}
try
{
var buf = new byte[chunkSize];
input.Read(buf, 0, chunkSize);
mdSink.Write(buf, 0, chunkSize);
}
catch (IOException e)
{
throw new IOException("Failed to read chunk #" + chunkIndex, e);
}
for (var i = 0; i < digestAlgorithmsArray.Length; i++)
{
var md = mds[i];
var concatenationOfChunkCountAndChunkDigests = digestsOfChunks[i];
var expectedDigestSizeBytes = digestOutputSizes[i];
md.TransformFinalBlock(new byte[0], 0, 0);
var hash = md.Hash;
var actualDigestSizeBytes = hash.Length;
if (actualDigestSizeBytes != expectedDigestSizeBytes)
{
throw new Exception(
"Unexpected output size of " + md
+ " digest: " + actualDigestSizeBytes);
}
Buffer.BlockCopy(hash, 0, concatenationOfChunkCountAndChunkDigests,
5 + chunkIndex * expectedDigestSizeBytes, hash.Length);
md.Dispose();
var digestAlgorithm = digestAlgorithmsArray[i];
var jcaAlgorithm = digestAlgorithm.JcaMessageDigestAlgorithm;
mds[i] = HashAlgorithm.Create(jcaAlgorithm);
}
inputOffset += chunkSize;
inputRemaining -= chunkSize;
chunkIndex++;
}
}
IDictionary <ContentDigestAlgorithm, byte[]> result =
new Dictionary <ContentDigestAlgorithm, byte[]>(digestAlgorithmsArray.Length);
for (var i = 0; i < digestAlgorithmsArray.Length; i++)
{
var digestAlgorithm = digestAlgorithmsArray[i];
var concatenationOfChunkCountAndChunkDigests = digestsOfChunks[i];
mds[i].Dispose();
byte[] hash;
using (var md = HashAlgorithm.Create(digestAlgorithm.JcaMessageDigestAlgorithm))
{
md.TransformFinalBlock(concatenationOfChunkCountAndChunkDigests, 0,
concatenationOfChunkCountAndChunkDigests.Length);
hash = md.Hash;
}
result.Add(digestAlgorithm, hash);
}
return(result);
}
