/// <summary>
/// Chunks and saves data to blobstore.
/// Operates on stream input, so Filestreams can be used and
/// entire files need not be loaded into memory.
/// If an error occurs (typically when reading from a stream
/// representing a file), it is thrown to the caller.
/// </summary>
/// <param name="inputstream"></param>
/// <param name="type"></param>
/// <param name="filehash"></param>
/// <param name="hashblobqueue"></param>
public static void SplitData(Stream inputstream, byte[] filehash, BlockingCollection <HashBlobPair> hashblobqueue)
{
// https://rsync.samba.org/tech_report/node3.html
List <byte> newblob = new();
byte[] alphachksum = new byte[2];
byte[] betachksum = new byte[2];
SHA1 sha1filehasher = HashTools.GetSHA1Hasher();
SHA1 sha1blobhasher = HashTools.GetSHA1Hasher();;
if (inputstream.Length != 0)
{
int readsize = 8_388_608;
int rollwindowsize = 32;
byte[] readin;
byte[] shifted = new byte[2];
for (int i = 0; i < inputstream.Length; i += readsize) // read the file in larger chunks for efficiency
{
if (i + readsize <= inputstream.Length) // readsize or more bytes left to read
{
readin = new byte[readsize];
inputstream.Read(readin, 0, readsize);
}
else // < readsize bytes left to read
{
readin = new byte[inputstream.Length % readsize];
inputstream.Read(readin, 0, (int)(inputstream.Length % readsize));
}
for (int j = 0; j < readin.Length; j++) // Byte by byte
{
newblob.Add(readin[j]);
HashTools.ByteSum(alphachksum, newblob[^ 1]);
/// <summary>
/// Chunks and saves data to blobstore.
/// Operates on stream input, so Filestreams can be used and
/// entire files need not be loaded into memory.
/// If an error occurs (typically when reading from a stream
/// representing a file), it is thrown to the caller.
/// </summary>
/// <param name="inputstream"></param>
/// <param name="type"></param>
/// <param name="filehash"></param>
/// <param name="hashblobqueue"></param>
public static void SplitData(Stream inputstream, byte[] filehash, BlockingCollection <HashBlobPair> hashblobqueue)
{
// https://rsync.samba.org/tech_report/node3.html
List <byte> newblob = new List <byte>();
byte[] alphachksum = new byte[2];
byte[] betachksum = new byte[2];
SHA1 sha1filehasher = HashTools.GetSHA1Hasher();
SHA1 sha1blobhasher = HashTools.GetSHA1Hasher();;
if (inputstream.Length != 0)
{
int readsize = 8_388_608;
int rollwindowsize = 32;
byte[] readin;
byte[] shifted = new byte[2];
for (int i = 0; i < inputstream.Length; i += readsize) // read the file in larger chunks for efficiency
{
if (i + readsize <= inputstream.Length) // readsize or more bytes left to read
{
readin = new byte[readsize];
inputstream.Read(readin, 0, readsize);
}
else // < readsize bytes left to read
{
readin = new byte[inputstream.Length % readsize];
inputstream.Read(readin, 0, (int)(inputstream.Length % readsize));
}
for (int j = 0; j < readin.Length; j++) // Byte by byte
{
newblob.Add(readin[j]);
HashTools.ByteSum(alphachksum, newblob[newblob.Count - 1]);
if (newblob.Count > rollwindowsize)
{
HashTools.ByteDifference(alphachksum, newblob[newblob.Count - rollwindowsize - 1]);
shifted[0] = (byte)((newblob[newblob.Count - 1] << 5) & 0xFF); // rollwindowsize = 32 = 2^5 => 5
shifted[1] = (byte)((newblob[newblob.Count - 1] >> 3) & 0xFF); // 8-5 = 3
HashTools.BytesDifference(betachksum, shifted);
}
HashTools.BytesSum(betachksum, alphachksum);
if (alphachksum[0] == 0xFF && betachksum[0] == 0xFF && betachksum[1] < 0x02) // (256*256*128)^-1 => expected value (/2) = ~4MB
{
byte[] blob = newblob.ToArray();
if (i + readsize >= inputstream.Length && j + 1 >= readin.Length) // Need to use TransformFinalBlock if at end of input
{
sha1filehasher.TransformFinalBlock(blob, 0, blob.Length);
}
else
{
sha1filehasher.TransformBlock(blob, 0, blob.Length, blob, 0);
}
hashblobqueue.Add(new HashBlobPair(sha1blobhasher.ComputeHash(blob), blob));
newblob = new List <byte>();
Array.Clear(alphachksum, 0, 2);
Array.Clear(betachksum, 0, 2);
}
}
}
if (newblob.Count != 0) // Create blob from remaining bytes
{
byte[] blob = newblob.ToArray();
sha1filehasher.TransformFinalBlock(blob, 0, blob.Length);
hashblobqueue.Add(new HashBlobPair(sha1blobhasher.ComputeHash(blob), blob));
}
}
else
{
byte[] blob = new byte[0];
sha1filehasher.TransformFinalBlock(blob, 0, blob.Length);
hashblobqueue.Add(new HashBlobPair(sha1blobhasher.ComputeHash(blob), blob));
}
Array.Copy(sha1filehasher.Hash, filehash, sha1filehasher.Hash.Length);
hashblobqueue.CompleteAdding();
}