public string GetHash(byte[] chunkdata)
{
HashTableHashing.SuperFastHashSimple sfh = new HashTableHashing.SuperFastHashSimple();
HashTableHashing.MurmurHash2Simple mm2h = new HashTableHashing.MurmurHash2Simple();
xxHash xxhash = new xxHash();
xxhash.Init();
xxhash.Update(chunkdata, chunkdata.Count());
string hash = "";
uint h1 = sfh.Hash(chunkdata);
uint h2 = mm2h.Hash(chunkdata);
uint h3 = xxhash.Digest();
BigInteger h = (BigInteger)h1 * (BigInteger)h2 * (BigInteger)h3;
hash = h.ToString() + h1.ToString() + h2.ToString() + h3.ToString();
// byte array representation of that string
byte[] encodedhash = new UTF8Encoding().GetBytes(hash);
// need MD5 to calculate the hash
byte[] md5hash = ((HashAlgorithm)CryptoConfig.CreateFromName("MD5")).ComputeHash(encodedhash);
// string representation (similar to UNIX format)
string encoded = BitConverter.ToString(md5hash)
// without dashes
.Replace("-", string.Empty)
// make lowercase
.ToLower();
return(encoded);
}
public static uint ComputeHash(FileInfo file)
{
xxHash hash = new xxHash();
hash.Init();
FileStream stream = new FileStream(file.FullName, FileMode.Open);
long bytesToRead = stream.Length;
long constSize = stream.Length;
byte[] buffer = new byte[4096];
long bytesRead = 0;
while (bytesToRead > 0)
{
if (Program.TERMINATE_BACKUP)
{
stream.Close();
return(0);
}
int len = stream.Read(buffer, 0, buffer.Length);
bytesRead += len;
bytesToRead -= len;
if (len == 0 && bytesToRead > 0)
{
//Unexpected IO
break;
}
BACKUP_PROGRESS_CURRENTFILE = (int)((bytesRead * 100) / constSize);
hash.Update(buffer, len);
}
stream.Close();
return(hash.Digest());
}
public static uint DetermineHashCode(Type type)
{
byte[] bytes = Encoding.UTF8.GetBytes(type.FullName);
xxHash hash = new xxHash();
hash.Init();
hash.Update(bytes, bytes.Length);
return(hash.Digest());
}
// TODO: for optimization, hardcode the whole header as static array
static byte[] CreateHeader()
{
var buff = new byte[4 + 3];
LittleEndianConverter.Write(MAGIC, buff, 0);
// Version 1; Block Independence
var flags = 1 << 6 | 1 << 5; //96;
// TODO: make lz4 block size configurable
// Block size 64Kb
var bd = 1 << 6; //64;
buff[4] = (byte)flags;
buff[5] = (byte)bd;
var hasher = new xxHash();
hasher.Init();
hasher.Update(buff, 6);
var checksum = hasher.Digest() >> 8 & 0xff; // 26
buff[6] = (byte)checksum;
return(buff);
}
//
// Implementation
//
bool ReadHeader()
{
// Read magic, FLG/BD and Descriptor Checksum
if (!StreamUtils.ReadAll(_base, _headerBuffer, 7))
{
return(false);
}
if (LittleEndianConverter.ReadUInt32(_headerBuffer, 0) != MAGIC)
{
throw new InvalidDataException("Invalid lz4 magic");
}
// parse FLG
var flg = _headerBuffer[4];
var version = flg >> 6;
if (version != 1)
{
throw new InvalidDataException($"Unsupported version of LZ4 format. Supported 1 but got {version}");
}
var hasBlockChecksum = (flg & (1 << 4)) != 0;
if (hasBlockChecksum)
{
throw new NotImplementedException("Block checksum is not implemented");
}
// parse BD and allocate uncompressed buffer
var bd = _headerBuffer[5];
var maxBlockSizeIndex = (bd >> 4) & 0x7;
if (maxBlockSizeIndex >= _maxBlockSizeTable.Length)
{
throw new InvalidDataException($"Invalid LZ4 max data block size index: {maxBlockSizeIndex}");
}
int maxBlockSize = _maxBlockSizeTable[maxBlockSizeIndex];
if (maxBlockSize == 0)
{
throw new InvalidDataException($"Invalid LZ4 max data block size index: {maxBlockSizeIndex}");
}
if (_uncompressedBuffer == null || _uncompressedBuffer.Length < maxBlockSize)
{
_uncompressedBuffer = new byte[Math.Max(maxBlockSize, _blockSize)];
}
_hasher.Init();
// Yep, this is the bug in kafka's framing checksum KAFKA-3160. Magic should not be checksummed but it is
_hasher.Update(_headerBuffer, 6); // Will need to patch it to accept offset in order to avoid unneeded reallocations,
// when want to exlude magic
var calculatedChecksum = (_hasher.Digest() >> 8) & 0xff;
if (calculatedChecksum != _headerBuffer[6])
{
throw new InvalidDataException("Lz4 Frame Descriptor checksum mismatch");
}
return(true);
}
internal void Finish()
{
_isClosed = true;
_checksum = (int)_hash.Digest();
}