public static ChunkedChecksum Compute(int size, System.IO.Stream stream, Action <long, long> progress = null)
{
ChunkedChecksum result = new ChunkedChecksum();
result.ChunkSize = size;
result.HashType = HashMode.Murmur3;
byte[] block = new byte[size];
List <Chunk> chunks = new List <Chunk>();
long offset = 0;
int index = 0;
System.Security.Cryptography.SHA1 sha1 = null;
if (result.HashType == HashMode.SHA1)
{
sha1 = System.Security.Cryptography.SHA1.Create();
}
while (true)
{
if (progress != null)
{
progress(size, offset);
}
int count = stream.Read(block, 0, size);
if (count == 0)
{
break;
}
Chunk chunk = new Chunk()
{
Index = index++,
Offset = offset,
Adler32 = FastHash(block, count),
//Hash = sha1.ComputeHash(block, 0, count),
Hash = new Utilities.Murmur3().ComputeHash(block, count),
Length = count
};
offset += size;
chunks.Add(chunk);
if (count < size)
{
break;
}
}
result.Chunks = chunks.ToArray();
result.ChunkCount = (uint)chunks.Count;
return(result);
}
internal static void Write(System.IO.Stream stream, ChunkedChecksum result)
{
if (result.HashType == HashMode.SHA1)
{
stream.Write(new byte[] { (byte)'h', (byte)'a', (byte)'s', (byte)'h' }, 0, 4);
}
else
{
stream.Write(new byte[] { (byte)'h', (byte)'s', (byte)'h', (byte)'2' }, 0, 4);
}
stream.Write(BitConverter.GetBytes(result.Chunks.Length), 0, 4);
stream.Write(BitConverter.GetBytes(result.ChunkSize), 0, 4);
stream.Write(BitConverter.GetBytes(result.ChunkCount), 0, 4);
stream.Write(BitConverter.GetBytes((int)result.HashType), 0, 4);
foreach (var x in result.Chunks)
{
stream.Write(BitConverter.GetBytes(x.Adler32), 0, 4);
stream.Write(x.Hash, 0, x.Hash.Length);
}
}
public static ChunkedChecksum Load(long filesize, System.IO.Stream stream)
{
ChunkedChecksum result = new ChunkedChecksum();
byte[] temp = new byte[16];
stream.Read(temp, 0, 16);
int version = 0;
if (temp[0] == 'h' && temp[1] == 'a' && temp[2] == 's' && temp[3] == 'h')
{
version = 1;
}
else if (temp[0] == 'h' && temp[1] == 's' && temp[2] == 'h' && temp[3] == '2')
{
version = 2;
}
if (version == 0)
{
throw new Exception();
}
result.ChunkSize = BitConverter.ToInt32(temp, 8);
result.ChunkCount = BitConverter.ToUInt32(temp, 12);
result.Chunks = new Chunk[result.ChunkCount];
if (version == 1)
{
result.HashType = HashMode.SHA1;
}
else if (version == 2)
{
stream.Read(temp, 0, 4);
result.HashType = (HashMode)BitConverter.ToInt32(temp, 0);
}
uint remaining = result.ChunkCount;
long offset = 0;
int index = 0;
while (remaining > 0)
{
stream.Read(temp, 0, 4);
Chunk c = new Chunk()
{
Adler32 = BitConverter.ToUInt32(temp, 0), Offset = offset
};
offset += result.ChunkSize;
if (remaining == 1)
{
c.Length = (int)(filesize - offset);
}
else
{
c.Length = result.ChunkSize;
}
if (result.HashType == HashMode.SHA1)
{
c.Hash = new byte[20];
stream.Read(c.Hash, 0, 20);
}
else if (result.HashType == HashMode.Murmur3)
{
c.Hash = new byte[16];
stream.Read(c.Hash, 0, 16);
}
c.Index = index;
result.Chunks[index++] = c;
remaining--;
}
return(result);
}
public static List <FileBlock> ComputeDelta(System.IO.Stream input, long inputSize, ChunkedChecksum chunks, out long deltaSize, Action <long, long> progress = null)
{
List <FileBlock> deltas = new List <FileBlock>();
if (inputSize < chunks.ChunkSize)
{
deltas.Add(new FileBlock()
{
Base = false, Offset = 0, Length = inputSize
});
deltaSize = 1 + 8 + inputSize;
return(deltas);
}
List <Chunk> sortedChunks = chunks.Chunks.OrderBy(x => new Tuple <uint, long>(x.Adler32, x.Offset)).ToList();
Dictionary <uint, int> adlerToIndex = new Dictionary <uint, int>();
for (int i = 0; i < sortedChunks.Count; i++)
{
adlerToIndex[sortedChunks[i].Adler32] = i;
}
BufferedView view = new BufferedView(input, 4 * 1024 * 1024);
CircularBuffer circularBuffer = new CircularBuffer(chunks.ChunkSize);
long readHead = 0;
long processHead = 0;
uint checksum = 0;
FileBlock lastMatch = null;
System.Security.Cryptography.SHA1 sha1 = null;
if (chunks.HashType == HashMode.SHA1)
{
sha1 = System.Security.Cryptography.SHA1.Create();
}
while (processHead != inputSize)
{
// first run through
if (readHead == processHead)
{
for (int i = 0; i < chunks.ChunkSize; i++)
{
circularBuffer.AddByte(view.Next());
}
readHead = chunks.ChunkSize;
checksum = FastHash(circularBuffer.ToArray(chunks.ChunkSize), chunks.ChunkSize);
}
int remainder = (int)(readHead - processHead);
if (progress != null)
{
progress(inputSize, processHead);
}
int possibleChunkLookup;
Chunk match = null;
if (adlerToIndex.TryGetValue(checksum, out possibleChunkLookup))
{
byte[] data = circularBuffer.ToArray(chunks.ChunkSize > remainder ? remainder : chunks.ChunkSize);
byte[] realHash = null;
if (sha1 != null)
{
realHash = sha1.ComputeHash(data, 0, data.Length);
}
else
{
realHash = (new Utilities.Murmur3()).ComputeHash(data);
}
while (true)
{
Chunk inspectedChunk = sortedChunks[possibleChunkLookup];
if (inspectedChunk.Adler32 != checksum)
{
break;
}
if (inspectedChunk.Length == remainder &&
System.Collections.StructuralComparisons.StructuralEqualityComparer.Equals(realHash, inspectedChunk.Hash))
{
match = inspectedChunk;
break;
}
possibleChunkLookup--;
}
}
if (match != null)
{
if (lastMatch == null || (lastMatch.Base == false || lastMatch.End != match.Offset))
{
lastMatch = new FileBlock()
{
Base = true, Length = 0, Offset = match.Offset
};
deltas.Add(lastMatch);
}
lastMatch.Length += match.Length;
}
else
{
if (lastMatch == null || lastMatch.Base == true)
{
lastMatch = new FileBlock()
{
Base = false, Length = 0, Offset = processHead
};
deltas.Add(lastMatch);
}
lastMatch.Length++;
}
// eat next byte
int bytesToEat = 1;
if (match != null)
{
bytesToEat = match.Length;
}
while (bytesToEat-- != 0)
{
remainder = (int)(readHead - processHead);
if (readHead < inputSize)
{
byte add = view.Next();
byte remove = circularBuffer.AddByte(add);
checksum = RotateHash(checksum, add, remove, remainder);
readHead++;
}
else if (bytesToEat == 0)
{
checksum = FastHash(circularBuffer.ToArray(remainder - 1), remainder - 1);
}
processHead++;
}
}
deltaSize = 0;
foreach (var x in deltas)
{
deltaSize++;
if (x.Base == false)
{
deltaSize += 8;
deltaSize += x.Length;
}
else
{
deltaSize += 8;
deltaSize += 8;
}
}
return(deltas);
}
