public static uint ComputeHash(FileInfo file)
{
xxHash hash = new xxHash();
hash.Init();
FileStream stream = new FileStream(file.FullName, FileMode.Open);
long bytesToRead = stream.Length;
long constSize = stream.Length;
byte[] buffer = new byte[4096];
long bytesRead = 0;
while (bytesToRead > 0)
{
if (Program.TERMINATE_BACKUP)
{
stream.Close();
return(0);
}
int len = stream.Read(buffer, 0, buffer.Length);
bytesRead += len;
bytesToRead -= len;
if (len == 0 && bytesToRead > 0)
{
//Unexpected IO
break;
}
BACKUP_PROGRESS_CURRENTFILE = (int)((bytesRead * 100) / constSize);
hash.Update(buffer, len);
}
stream.Close();
return(hash.Digest());
}
public static long GenerateRequestHash(byte[] authTicket, byte[] request)
{
var seed = BitConverter.ToUInt64(new xxHash(64, 0x1B845238).ComputeHash(authTicket), 0);
var xxh64 = new xxHash(64, seed);
return(BitConverter.ToInt64(xxh64.ComputeHash(request), 0));
}
public string GetHash(byte[] chunkdata)
{
HashTableHashing.SuperFastHashSimple sfh = new HashTableHashing.SuperFastHashSimple();
HashTableHashing.MurmurHash2Simple mm2h = new HashTableHashing.MurmurHash2Simple();
xxHash xxhash = new xxHash();
xxhash.Init();
xxhash.Update(chunkdata, chunkdata.Count());
string hash = "";
uint h1 = sfh.Hash(chunkdata);
uint h2 = mm2h.Hash(chunkdata);
uint h3 = xxhash.Digest();
BigInteger h = (BigInteger)h1 * (BigInteger)h2 * (BigInteger)h3;
hash = h.ToString() + h1.ToString() + h2.ToString() + h3.ToString();
// byte array representation of that string
byte[] encodedhash = new UTF8Encoding().GetBytes(hash);
// need MD5 to calculate the hash
byte[] md5hash = ((HashAlgorithm)CryptoConfig.CreateFromName("MD5")).ComputeHash(encodedhash);
// string representation (similar to UNIX format)
string encoded = BitConverter.ToString(md5hash)
// without dashes
.Replace("-", string.Empty)
// make lowercase
.ToLower();
return(encoded);
}
public void ComputeHash()
{
try
{
string firma_base64 = "";
xxHash firma = null;
if (firma == null)
{
firma = new xxHash();
}
//using (MD5 firma = MD5.Create())
using (FileStream streamFichero = File.OpenRead(basename))
{
byte[] f = firma.ComputeHash(streamFichero);
firma_base64 = Convert.ToBase64String(f);
}
hash = firma_base64;
}
catch
{
throw;
}
}
public static byte[] XXHash(string stringToHash)
{
byte[] hash;
xxHash xx = new xxHash();
hash = xx.ComputeHash(Encoding.ASCII.GetBytes(stringToHash));
return(hash);
}
public static string ComputeFileHash(string filePath)
{
using (var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read))
{
IEnumerable <string> enumerable = new xxHash(Seed).ComputeHash(fileStream).Select(b => b.ToString("X2")).Reverse();
return(string.Join("", enumerable.ToArray()).ToLower().TrimStart('0'));
}
}
public static byte[] ComputeHash(byte[] buffer, int offset, int length)
{
var xxHash = new xxHash(Seed);
using (var memoryStream = new MemoryStream(buffer, offset, length))
{
return(xxHash.ComputeHash(memoryStream));
}
}
public static uint DetermineHashCode(Type type)
{
byte[] bytes = Encoding.UTF8.GetBytes(type.FullName);
xxHash hash = new xxHash();
hash.Init();
hash.Update(bytes, bytes.Length);
return(hash.Digest());
}
public static uint GenerateLocation2(double lat, double lng, double alt)
{
var xxh32 = new xxHash(32, 0x1B845238);
var locationBytes = new List <byte>();
locationBytes.AddRange(BitConverter.GetBytes(lat).Reverse());
locationBytes.AddRange(BitConverter.GetBytes(lng).Reverse());
locationBytes.AddRange(BitConverter.GetBytes(alt).Reverse());
return(BitConverter.ToUInt32(xxh32.ComputeHash(locationBytes.ToArray()), 0));
}
public static uint GenerateLocation1(byte[] authTicket, double lat, double lng, double alt, ulong hashSeed)
{
var seed = BitConverter.ToUInt32(new xxHash(32, hashSeed).ComputeHash(authTicket), 0);
var xxh32 = new xxHash(32, seed);
var locationBytes = new List <byte>();
locationBytes.AddRange(BitConverter.GetBytes(lat).Reverse());
locationBytes.AddRange(BitConverter.GetBytes(lng).Reverse());
locationBytes.AddRange(BitConverter.GetBytes(alt).Reverse());
return(BitConverter.ToUInt32(xxh32.ComputeHash(locationBytes.ToArray()), 0));
}
protected override async Task <AsciiMaxStringReference> AssignUpsertedReferences(AsciiMaxStringReference record)
{
var bytes = new xxHash(64).ComputeHash(Encoding.UTF8.GetBytes(record.Value));
var hash = new AsciiStringReference {
Value = Convert.ToBase64String(bytes)
};
hash = await _strings.UpsertAsync(hash);
record.Hash = hash;
record.HashId = hash.AsciiStringReferenceId;
return(record);
}
public HttpContent AsContent(Command cmd, IServerContext context, bool multipart)
{
_data.Position = 0;
HttpContent result;
#if FILEIO
if (_data == null)
{
result = new SimpleContent(new FileStream(_filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 4096));
}
else
{
result = new SimpleContent(_data);
}
#else
result = new SimpleContent(_data);
#endif
var id = _id[0] == '@' ? cmd.Substitute(_id, context) : _id;
var path = GetFileName(_path[0] == '@' ? cmd.Substitute(_path, context) : _path);
if (multipart)
{
result.Headers.Add("Content-Disposition", string.Format("form-data; name=\"{0}\"; filename=\"{1}\"", id, path));
}
else
{
result.Headers.Add("Content-Disposition", "attachment; filename*=" + Encode5987(path));
result.Headers.Add("Content-Range", string.Format("bytes {0}-{1}/{2}", 0, _length - 1, _length));
var hash = default(byte[]);
_data.Position = 0;
#if FILEIO
if (_data == null)
{
hash = new xxHash(32).ComputeHash(new FileStream(_filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 4096));
}
else
{
hash = new xxHash(32).ComputeHash(_data);
}
#else
hash = new xxHash(32).ComputeHash(_data);
#endif
_data.Position = 0;
var hashStr = BitConverter.ToUInt32(hash, 0).ToString(CultureInfo.InvariantCulture);
result.Headers.Add("Aras-Content-Range-Checksum", hashStr);
result.Headers.Add("Aras-Content-Range-Checksum-Type", "xxHashAsUInt32AsDecimalString");
}
return(result);
}
private static void HashFiles(List <List <FileDetails> > sizeGroups, QueueStatus status, bool showProgress, bool useFastHash)
{
IFileHash hash;
if (useFastHash)
{
hash = new xxHash();
}
else
{
hash = new SHA1Hash();
}
//Hash all dupes in each group
foreach (List <FileDetails> sizeGroup in sizeGroups)
{
foreach (FileDetails details in sizeGroup)
{
if (TryReadFile(details, out byte[] data))
/// <summary>
/// Computes a 64-bit hash for a file.
/// </summary>
/// <param name="file">File to hash</param>
/// <returns>ulong hash or null if the hash failed.</returns>
private static ulong?GetHash(FileInfo file)
{
byte[] bhash;
var xh = new xxHash(64);
try {
using (var fs = file.OpenRead()) {
bhash = xh.ComputeHash(fs);
}
}
catch (IOException) {
return(null);
}
ulong lhash = 0;
for (var i = 0; i < bhash.Length; i++)
{
lhash |= (ulong)bhash[i] << (8 * i);
}
return(lhash);
}
public string ComputeHash(string Filename)
{
try
{
string firma_base64 = "";
xxHash firma = new xxHash(64);;
using (FileStream streamFichero = File.OpenRead(Filename))
{
byte[] f = firma.ComputeHash(streamFichero);
firma_base64 = Convert.ToBase64String(f);
}
return(firma_base64);
}
catch
{
throw;
}
}
public async Task <string> ComputeHashAsync(string Filename)
{
try
{
string firma_base64 = "";
xxHash firma = new xxHash(64);;
using (FileStream streamFichero = File.OpenRead(Filename))
{
byte[] f = await firma.ComputeHashAsync(streamFichero);
firma_base64 = Convert.ToBase64String(f);
}
return(firma_base64);
}
catch (Exception x)
{
throw x;
}
}
// TODO: for optimization, hardcode the whole header as static array
static byte[] CreateHeader()
{
var buff = new byte[4 + 3];
LittleEndianConverter.Write(MAGIC, buff, 0);
// Version 1; Block Independence
var flags = 1 << 6 | 1 << 5; //96;
// TODO: make lz4 block size configurable
// Block size 64Kb
var bd = 1 << 6; //64;
buff[4] = (byte)flags;
buff[5] = (byte)bd;
var hasher = new xxHash();
hasher.Init();
hasher.Update(buff, 6);
var checksum = hasher.Digest() >> 8 & 0xff; // 26
buff[6] = (byte)checksum;
return(buff);
}
private Unknown6 GenerateSignature(IEnumerable <IMessage> requests)
{
var sig = new Signature();
sig.TimestampSinceStart = (ulong)InternalWatch.ElapsedMilliseconds;
sig.Timestamp = (ulong)DateTime.UtcNow.ToUnixTime();
sig.SensorInfo = new Signature.Types.SensorInfo
{
AccelNormalizedZ = GenRandom(9.8),
AccelNormalizedX = GenRandom(0.02),
AccelNormalizedY = GenRandom(0.3),
TimestampSnapshot = (ulong)(InternalWatch.ElapsedMilliseconds - 230L),
MagnetometerX = GenRandom(12271042913198472.0),
MagnetometerY = GenRandom(-0.015570580959320068),
MagnetometerZ = GenRandom(0.010850906372070313),
AngleNormalizedX = GenRandom(17.950439453125),
AngleNormalizedY = GenRandom(-23.36273193359375),
AngleNormalizedZ = GenRandom(-48.8250732421875),
AccelRawX = GenRandom(-0.0120010357350111),
AccelRawY = GenRandom(-0.04214850440621376),
AccelRawZ = GenRandom(0.94571763277053833),
GyroscopeRawX = GenRandom(7.62939453125E-05),
GyroscopeRawY = GenRandom(-0.00054931640625),
GyroscopeRawZ = GenRandom(0.0024566650390625),
AccelerometerAxes = 3uL
};
sig.DeviceInfo = new Signature.Types.DeviceInfo
{
DeviceId = (_settings.DeviceId ?? $"1{RandomDevice.Next(12345)}"),
AndroidBoardName = (_settings.AndroidBoardName ?? $"plsnobanerino{RandomDevice.Next(12345)}"),
AndroidBootloader = (_settings.AndroidBootloader ?? $"andriod{RandomDevice.Next(12345)}"),
DeviceBrand = (_settings.DeviceBrand ?? $"samsung{RandomDevice.Next(12345)}"),
DeviceModel = (_settings.DeviceModel ?? $"1.0{RandomDevice.Next(12345)}"),
DeviceModelIdentifier = (_settings.DeviceModelIdentifier ?? $"1.0{RandomDevice.Next(12345)}"),
DeviceModelBoot = (_settings.DeviceModelBoot ?? $"1234{RandomDevice.Next(12345)}"),
HardwareManufacturer = (_settings.HardwareManufacturer ?? $"sprint{RandomDevice.Next(12345)}"),
HardwareModel = (_settings.HardwareModel ?? $"thisone{RandomDevice.Next(12345)}"),
FirmwareBrand = (_settings.FirmwareBrand ?? $"thatone{RandomDevice.Next(12345)}"),
FirmwareTags = (_settings.FirmwareTags ?? $"123{RandomDevice.Next(12345)}"),
FirmwareType = (_settings.FirmwareType ?? $"n{RandomDevice.Next(12345)}o"),
FirmwareFingerprint = (_settings.FirmwareFingerprint ?? $"kth{RandomDevice.Next(12345)}x")
};
sig.LocationFix.Add(new Signature.Types.LocationFix
{
Provider = "network",
Latitude = (float)_latitude,
Longitude = (float)_longitude,
Altitude = (float)_altitude,
TimestampSinceStart = (ulong)(InternalWatch.ElapsedMilliseconds - 200L),
Floor = 3u,
LocationType = 1uL
});
var x = new xxHash(32, 461656632uL);
var firstHash = BitConverter.ToUInt32(x.ComputeHash(_authTicket.ToByteArray()), 0);
x = new xxHash(32, (ulong)firstHash);
var locationBytes = BitConverter.GetBytes(_latitude).Reverse <byte>().Concat(BitConverter.GetBytes(_longitude).Reverse <byte>()).Concat(BitConverter.GetBytes(_altitude).Reverse <byte>()).ToArray <byte>();
sig.LocationHash1 = BitConverter.ToUInt32(x.ComputeHash(locationBytes), 0);
x = new xxHash(32, 461656632uL);
sig.LocationHash2 = BitConverter.ToUInt32(x.ComputeHash(locationBytes), 0);
x = new xxHash(64, 461656632uL);
var seed = BitConverter.ToUInt64(x.ComputeHash(_authTicket.ToByteArray()), 0);
x = new xxHash(64, seed);
foreach (var req in requests)
{
sig.RequestHash.Add(BitConverter.ToUInt64(x.ComputeHash(req.ToByteArray()), 0));
}
sig.Unk22 = ByteString.CopyFrom(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
var val = new Unknown6
{
RequestType = 6,
Unknown2 = new Unknown6.Types.Unknown2 {
Unknown1 = ByteString.CopyFrom(Encrypt(sig.ToByteArray()))
}
};
return(val);
}
private Unknown6 GenerateSignature(IEnumerable <IMessage> requests)
{
var sig = new Signature();
sig.TimestampSinceStart = (ulong)_internalWatch.ElapsedMilliseconds;
sig.Timestamp = (ulong)DateTime.UtcNow.ToUnixTime();
sig.SensorInfo = new Signature.Types.SensorInfo
{
AccelNormalizedZ = GenRandom(9.8),
AccelNormalizedX = GenRandom(0.02),
AccelNormalizedY = GenRandom(0.3),
TimestampSnapshot = (ulong)_internalWatch.ElapsedMilliseconds - 230,
MagnetometerX = GenRandom(0.12271042913198471),
MagnetometerY = GenRandom(-0.015570580959320068),
MagnetometerZ = GenRandom(0.010850906372070313),
AngleNormalizedX = GenRandom(17.950439453125),
AngleNormalizedY = GenRandom(-23.36273193359375),
AngleNormalizedZ = GenRandom(-48.8250732421875),
AccelRawX = GenRandom(-0.0120010357350111),
AccelRawY = GenRandom(-0.04214850440621376),
AccelRawZ = GenRandom(0.94571763277053833),
GyroscopeRawX = GenRandom(7.62939453125e-005),
GyroscopeRawY = GenRandom(-0.00054931640625),
GyroscopeRawZ = GenRandom(0.0024566650390625),
AccelerometerAxes = 3
};
sig.DeviceInfo = new Signature.Types.DeviceInfo
{
DeviceId = settings.DeviceId,
AndroidBoardName = settings.AndroidBoardName,
AndroidBootloader = settings.AndroidBootloader,
DeviceBrand = settings.DeviceBrand,
DeviceModel = settings.DeviceModel,
DeviceModelIdentifier = settings.DeviceModelIdentifier,
DeviceModelBoot = settings.DeviceModelBoot,
HardwareManufacturer = settings.HardwareManufacturer,
HardwareModel = settings.HardwareModel,
FirmwareBrand = settings.FirmwareBrand,
FirmwareTags = settings.FirmwareTags,
FirmwareType = settings.FirmwareType,
FirmwareFingerprint = settings.FirmwareFingerprint
};
sig.LocationFix.Add(new Signature.Types.LocationFix
{
Provider = "network",
//Unk4 = 120,
Latitude = (float)_latitude,
Longitude = (float)_longitude,
Altitude = (float)_altitude,
TimestampSinceStart = (ulong)_internalWatch.ElapsedMilliseconds - 200,
Floor = 3,
LocationType = 1
});
//Compute 10
var x = new xxHash(32, 0x1B845238);
var firstHash = BitConverter.ToUInt32(x.ComputeHash(_authTicket.ToByteArray()), 0);
x = new xxHash(32, firstHash);
var locationBytes = BitConverter.GetBytes(_latitude).Reverse()
.Concat(BitConverter.GetBytes(_longitude).Reverse())
.Concat(BitConverter.GetBytes(_altitude).Reverse()).ToArray();
sig.LocationHash1 = BitConverter.ToUInt32(x.ComputeHash(locationBytes), 0);
//Compute 20
x = new xxHash(32, 0x1B845238);
sig.LocationHash2 = BitConverter.ToUInt32(x.ComputeHash(locationBytes), 0);
//Compute 24
x = new xxHash(64, 0x1B845238);
var seed = BitConverter.ToUInt64(x.ComputeHash(_authTicket.ToByteArray()), 0);
x = new xxHash(64, seed);
foreach (var req in requests)
{
sig.RequestHash.Add(BitConverter.ToUInt64(x.ComputeHash(req.ToByteArray()), 0));
}
//static for now
sig.Unk22 = ByteString.CopyFrom(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C,
0x0D, 0x0E, 0x0F);
var val = new Unknown6();
val.RequestType = 6;
val.Unknown2 = new Unknown6.Types.Unknown2();
val.Unknown2.EncryptedSignature = ByteString.CopyFrom(Encrypt(sig.ToByteArray()));
return(val);
}
public void xxHash64(string a, ulong hash)
{
IFileHash h = new xxHash();
Assert.Equal(hash, h.GetHash(Encoding.Unicode.GetBytes(a)));
}
public IEnumerable <HttpContent> AsContent(Command cmd, IServerContext context, bool multipart)
{
var stream = _data;
if (stream?.CanSeek == true)
{
stream.Position = 0;
}
var disposeLast = false;
#if FILEIO
if (stream == null)
{
stream = new FileStream(_filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 4096);
disposeLast = true;
}
#endif
const long chunkSize = 16 * 1024 * 1024;
var numChunks = _length.HasValue && !multipart ? (int)Math.Ceiling((double)_length / chunkSize) : 1;
var results = new HttpContent[numChunks];
for (var i = 0; i < numChunks; i++)
{
var chunkLength = _length;
if (_length.HasValue && numChunks > 1)
{
chunkLength = Math.Min(chunkSize, _length.Value - i * chunkSize);
results[i] = new FileStreamContent(stream, chunkLength.Value, disposeLast && i == numChunks - 1);
}
else
{
results[i] = new SimpleContent(stream, disposeLast);
}
var id = _id[0] == '@' ? cmd.Substitute(_id, context) : _id;
var path = GetFileName(_path[0] == '@' ? cmd.Substitute(_path, context) : _path);
if (multipart)
{
results[i].Headers.Add("Content-Disposition", string.Format("form-data; name=\"{0}\"; filename=\"{1}\"", id, path));
}
else
{
results[i].Headers.Add("Content-Disposition", "attachment; filename*=" + Encode5987(path));
if (chunkLength.HasValue && chunkLength.Value > 0)
{
results[i].Headers.Add("Content-Range", string.Format("bytes {0}-{1}/{2}", i * chunkSize, i * chunkSize + chunkLength - 1, _length));
}
if (numChunks == 1)
{
var hash = default(byte[]);
#if FILEIO
if (_data == null)
{
using (var s = new FileStream(_filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 4096))
hash = new xxHash(32).ComputeHash(s);
}
else if (_data.CanSeek)
{
_data.Position = 0;
hash = new xxHash(32).ComputeHash(_data);
_data.Position = 0;
}
#else
if (_data.CanSeek)
{
_data.Position = 0;
hash = new xxHash(32).ComputeHash(_data);
_data.Position = 0;
}
#endif
if (hash != null)
{
var hashStr = BitConverter.ToUInt32(hash, 0).ToString(CultureInfo.InvariantCulture);
results[i].Headers.Add("Aras-Content-Range-Checksum", hashStr);
results[i].Headers.Add("Aras-Content-Range-Checksum-Type", "xxHashAsUInt32AsDecimalString");
}
}
}
}
return(results);
}
