public DeltaStream(Stream signatureStream, Stream inputStream)
{
SignatureFile signatures = SignatureHelpers.ParseSignatureFile(signatureStream);
IEnumerable <OutputCommand> enumerable = DeltaCalculator.ComputeCommands(new BinaryReader(inputStream), signatures);
commandsToOutput = enumerable.GetEnumerator();
currentCommandStream = new MemoryStream();
BinaryWriter binaryWriter = new BinaryWriter(currentCommandStream);
StreamHelpers.WriteBigEndian(binaryWriter, 1920139830uL);
binaryWriter.Flush();
currentCommandStream.Seek(0L, SeekOrigin.Begin);
}
public static SignatureFile ParseSignatureFile(Stream s)
{
var result = new SignatureFile();
var r = new BinaryReader(s);
uint magicNumber = StreamHelpers.ReadBigEndianUint32(r);
if (magicNumber == (uint)MagicNumber.Blake2Signature)
{
result.StrongSumMethod = CalculateBlake2StrongSum;
}
else if (magicNumber == (uint)MagicNumber.Md5MagicNumber)
{
result.StrongSumMethod = CalculateMD5StrongSum;
}
else if (magicNumber == (uint)MagicNumber.MurMur3)
{
result.StrongSumMethod = CalculateMurmur3;
}
else
{
throw new InvalidDataException(string.Format("Unknown magic number {0}", magicNumber));
}
result.BlockLength = (int)StreamHelpers.ReadBigEndianUint32(r);
result.StrongSumLength = (int)StreamHelpers.ReadBigEndianUint32(r);
var signatures = new List<BlockSignature>();
ulong i = 0;
while (true)
{
byte[] weakSumBytes = r.ReadBytes(4);
if (weakSumBytes.Length == 0)
{
// we're at the end of the file
break;
}
int weakSum = (int)StreamHelpers.ConvertFromBigEndian(weakSumBytes);
byte[] strongSum = r.ReadBytes(result.StrongSumLength);
signatures.Add(new BlockSignature
{
StartPos = (ulong)result.BlockLength * i,
WeakSum = weakSum,
StrongSum = strongSum
});
i++;
}
result.BlockLookup = signatures.ToLookup(sig => sig.WeakSum);
return result;
}
public static SignatureFile ParseSignatureFile(Stream s)
{
var result = new SignatureFile();
var r = new BinaryReader(s);
uint magicNumber = StreamHelpers.ReadBigEndianUint32(r);
if (magicNumber == (uint)MagicNumber.Blake2Signature)
{
result.StrongSumMethod = CalculateBlake2StrongSum;
}
else
{
throw new InvalidDataException(string.Format("Unknown magic number {0}", magicNumber));
}
result.BlockLength = (int)StreamHelpers.ReadBigEndianUint32(r);
result.StrongSumLength = (int)StreamHelpers.ReadBigEndianUint32(r);
var signatures = new List <BlockSignature>();
ulong i = 0;
while (true)
{
byte[] weakSumBytes = r.ReadBytes(4);
if (weakSumBytes.Length == 0)
{
// we're at the end of the file
break;
}
int weakSum = (int)StreamHelpers.ConvertFromBigEndian(weakSumBytes);
byte[] strongSum = r.ReadBytes(result.StrongSumLength);
signatures.Add(new BlockSignature
{
StartPos = (ulong)result.BlockLength * i,
WeakSum = weakSum,
StrongSum = strongSum
});
i++;
}
result.BlockLookup = signatures.ToLookup(sig => sig.WeakSum);
return(result);
}
public static SignatureFile ParseSignatureFile(Stream s)
{
SignatureFile signatureFile = default(SignatureFile);
BinaryReader binaryReader = new BinaryReader(s);
uint num = StreamHelpers.ReadBigEndianUint32(binaryReader);
if (num != 1920139575)
{
throw new InvalidDataException($"Unknown magic number {num}");
}
signatureFile.StrongSumMethod = CalculateBlake2StrongSum;
signatureFile.BlockLength = (int)StreamHelpers.ReadBigEndianUint32(binaryReader);
signatureFile.StrongSumLength = (int)StreamHelpers.ReadBigEndianUint32(binaryReader);
Dictionary <int, List <BlockSignature> > dictionary = new Dictionary <int, List <BlockSignature> >();
ulong num2 = 0uL;
while (true)
{
byte[] array = binaryReader.ReadBytes(4);
if (array.Length == 0)
{
break;
}
int num3 = (int)StreamHelpers.ConvertFromBigEndian(array);
byte[] strongSum = binaryReader.ReadBytes(signatureFile.StrongSumLength);
List <BlockSignature> value = null;
if (!dictionary.TryGetValue(num3, out value))
{
value = new List <BlockSignature>();
dictionary.Add(num3, value);
}
value.Add(new BlockSignature
{
StartPos = (ulong)((long)signatureFile.BlockLength * (long)num2),
WeakSum = num3,
StrongSum = strongSum
});
num2++;
}
signatureFile.BlockLookup = dictionary;
return(signatureFile);
}
public static IEnumerable<OutputCommand> ComputeCommands(
BinaryReader inputStream,
SignatureFile signatures)
{
OutputCommand currentCommand = new OutputCommand { Kind = CommandKind.Reserved };
Rollsum currentSum;
Queue<byte> currentBlock;
// Read the first block to initialize
ReadNewBlock(inputStream, signatures.BlockLength, out currentBlock, out currentSum);
while (currentBlock.Count > 0)
{
// if the block has a matching rolling sum to any contained in the signature, those are candidates
var matchCandidates = signatures.BlockLookup[currentSum.Digest];
bool wasMatch = false;
if (matchCandidates.Any())
{
// now compute the strong sum and see if any of the signature blocks match it
byte[] currentStrongSum = signatures.StrongSumMethod(currentBlock.ToArray()).Take(signatures.StrongSumLength).ToArray();
var matchingBlock = matchCandidates.FirstOrDefault(s => s.StrongSum.SequenceEqual(currentStrongSum));
// if one of the signature blocks was a match, we can generate a Copy command
if (matchingBlock != null)
{
if (currentCommand.Kind == CommandKind.Copy &&
currentCommand.Position + currentCommand.Length == matchingBlock.StartPos)
{
// in this case, we can just add this copy onto the previous one
currentCommand.Length = currentCommand.Length + (ulong)currentBlock.Count;
}
else
{
// if we can't append to the current command, then return the current command and start a new one
if (currentCommand.Kind != CommandKind.Reserved)
{
yield return currentCommand;
}
currentCommand = new OutputCommand
{
Kind = CommandKind.Copy,
Position = matchingBlock.StartPos,
Length = (ulong)currentBlock.Count
};
}
// if we found a match, then we read a whole new block and reset the sum to it's hash
wasMatch = true;
ReadNewBlock(inputStream, signatures.BlockLength, out currentBlock, out currentSum);
}
}
// if there was no match for the current block, the we have to output it's first byte (at least)
// as a literal value, and try again starting at the next byte
if (!wasMatch)
{
// pull out the oldest byte
byte oldestByte = currentBlock.Dequeue();
// if we're not at the end of the file
if (inputStream.BaseStream.Position != inputStream.BaseStream.Length)
{
// read a byte and add it in, hoping to find a match at the next spot
byte nextByte = inputStream.ReadByte();
currentBlock.Enqueue(nextByte);
currentSum.Rotate(byteOut: oldestByte, byteIn: nextByte);
}
else
{
// if we are at the end of the file, then just rollout the oldestByte and see if we have a match for the partial block
// (this could happen if we match the last block in the original file)
currentSum.Rollout(byteOut: oldestByte);
}
if (currentCommand.Kind == CommandKind.Literal && currentCommand.Literal.Count < MaxLiteralLength)
{
// if we already have a literal command, just append a new byte on
currentCommand.Literal.Add(oldestByte);
}
else
{
// otherwise we have to emit the current command and start a new literal command
if (currentCommand.Kind != CommandKind.Reserved)
{
yield return currentCommand;
}
currentCommand = new OutputCommand
{
Kind = CommandKind.Literal,
Literal = new List<byte> { oldestByte },
};
}
}
}
yield return currentCommand;
yield return new OutputCommand
{
Kind = CommandKind.End,
};
}
public static IEnumerable <OutputCommand> ComputeCommands(
BinaryReader inputStream,
SignatureFile signatures)
{
OutputCommand currentCommand = new OutputCommand {
Kind = CommandKind.Reserved
};
Rollsum currentSum;
Queue <byte> currentBlock;
// Read the first block to initialize
ReadNewBlock(inputStream, signatures.BlockLength, out currentBlock, out currentSum);
while (currentBlock.Count > 0)
{
// if the block has a matching rolling sum to any contained in the signature, those are candidates
var matchCandidates = signatures.BlockLookup[currentSum.Digest];
bool wasMatch = false;
if (matchCandidates.Any())
{
// now compute the strong sum and see if any of the signature blocks match it
byte[] currentStrongSum = signatures.StrongSumMethod(currentBlock.ToArray()).Take(signatures.StrongSumLength).ToArray();
var matchingBlock = matchCandidates.FirstOrDefault(s => s.StrongSum.SequenceEqual(currentStrongSum));
// if one of the signature blocks was a match, we can generate a Copy command
if (matchingBlock != null)
{
if (currentCommand.Kind == CommandKind.Copy &&
currentCommand.Position + currentCommand.Length == matchingBlock.StartPos)
{
// in this case, we can just add this copy onto the previous one
currentCommand.Length = currentCommand.Length + (ulong)currentBlock.Count;
}
else
{
// if we can't append to the current command, then return the current command and start a new one
if (currentCommand.Kind != CommandKind.Reserved)
{
yield return(currentCommand);
}
currentCommand = new OutputCommand
{
Kind = CommandKind.Copy,
Position = matchingBlock.StartPos,
Length = (ulong)currentBlock.Count
};
}
// if we found a match, then we read a whole new block and reset the sum to it's hash
wasMatch = true;
ReadNewBlock(inputStream, signatures.BlockLength, out currentBlock, out currentSum);
}
}
// if there was no match for the current block, the we have to output it's first byte (at least)
// as a literal value, and try again starting at the next byte
if (!wasMatch)
{
// pull out the oldest byte
byte oldestByte = currentBlock.Dequeue();
// if we're not at the end of the file
if (inputStream.BaseStream.Position != inputStream.BaseStream.Length)
{
// read a byte and add it in, hoping to find a match at the next spot
byte nextByte = inputStream.ReadByte();
currentBlock.Enqueue(nextByte);
currentSum.Rotate(byteOut: oldestByte, byteIn: nextByte);
}
else
{
// if we are at the end of the file, then just rollout the oldestByte and see if we have a match for the partial block
// (this could happen if we match the last block in the original file)
currentSum.Rollout(byteOut: oldestByte);
}
if (currentCommand.Kind == CommandKind.Literal && currentCommand.Literal.Count < MaxLiteralLength)
{
// if we already have a literal command, just append a new byte on
currentCommand.Literal.Add(oldestByte);
}
else
{
// otherwise we have to emit the current command and start a new literal command
if (currentCommand.Kind != CommandKind.Reserved)
{
yield return(currentCommand);
}
currentCommand = new OutputCommand
{
Kind = CommandKind.Literal,
Literal = new List <byte> {
oldestByte
},
};
}
}
}
yield return(currentCommand);
yield return(new OutputCommand
{
Kind = CommandKind.End,
});
}
public static IEnumerable <OutputCommand> ComputeCommands(BinaryReader inputStream, SignatureFile signatures)
{
OutputCommand outputCommand = default(OutputCommand);
outputCommand.Kind = CommandKind.Reserved;
OutputCommand outputCommand2 = outputCommand;
long inputLength = inputStream.BaseStream.Length;
ReadNewBlock(inputStream, signatures.BlockLength, out Queue <byte> currentBlock, out Rollsum currentSum);
while (currentBlock.Count > 0)
{
List <BlockSignature> value = null;
bool flag = false;
if (signatures.BlockLookup.TryGetValue(currentSum.Digest, out value))
{
byte[] currentStrongSum = signatures.StrongSumMethod(currentBlock.ToArray());
BlockSignature matchingBlock = FindBlockSignatureByStrongSum(value, currentStrongSum);
if (matchingBlock != null)
{
if (outputCommand2.Kind == CommandKind.Copy && outputCommand2.Position + outputCommand2.Length == matchingBlock.StartPos)
{
outputCommand2.Length = (ulong)((long)outputCommand2.Length + (long)currentBlock.Count);
}
else
{
if (outputCommand2.Kind != CommandKind.Reserved)
{
yield return(outputCommand2);
}
outputCommand = default(OutputCommand);
outputCommand.Kind = CommandKind.Copy;
outputCommand.Position = matchingBlock.StartPos;
outputCommand.Length = (ulong)currentBlock.Count;
outputCommand2 = outputCommand;
}
flag = true;
ReadNewBlock(inputStream, signatures.BlockLength, out currentBlock, out currentSum);
}
}
if (!flag)
{
byte oldestByte = currentBlock.Dequeue();
if (inputStream.BaseStream.Position != inputLength)
{
byte b = inputStream.ReadByte();
currentBlock.Enqueue(b);
currentSum.Rotate(oldestByte, b);
}
else
{
currentSum.Rollout(oldestByte);
}
if (outputCommand2.Kind == CommandKind.Literal && outputCommand2.Literal.Count < 4194304)
{
outputCommand2.Literal.Add(oldestByte);
}
else
{
if (outputCommand2.Kind != CommandKind.Reserved)
{
yield return(outputCommand2);
}
outputCommand = default(OutputCommand);
outputCommand.Kind = CommandKind.Literal;
outputCommand.Literal = new List <byte>
{
oldestByte
};
outputCommand2 = outputCommand;
}
}
}
yield return(outputCommand2);
outputCommand = new OutputCommand
{
Kind = CommandKind.End
};
yield return(outputCommand);
}