public (string, BackupRecord) GetBackupHashAndRecord(string bsname, string prefix, int offset = 0) { var bset = LoadBackupSet(bsname); var match = HashByPrefix(bsname, prefix); if (match == null || match.Value.multiplematches == true) { // TODO: throw this exception out of HashByPrefix? throw new KeyNotFoundException(); } int pidx = 0; for (int i = 0; i < bset.Backups.Count; i++) { if (bset.Backups[i].hash.SequenceEqual(match.Value.singlematchhash)) { pidx = i; break; } } int bidx = pidx + offset; if (bidx >= 0 && bidx < bset.Backups.Count) { return(HashTools.ByteArrayToHexViaLookup32(bset.Backups[bidx].hash).ToLower(), GetBackupRecord(bsname, bset.Backups[bidx].hash)); } else { throw new IndexOutOfRangeException(); } }
/// <summary> /// Chunks and saves data to blobstore. /// Operates on stream input, so Filestreams can be used and /// entire files need not be loaded into memory. /// If an error occurs (typically when reading from a stream /// representing a file), it is thrown to the caller. /// </summary> /// <param name="inputstream"></param> /// <param name="type"></param> /// <param name="filehash"></param> /// <param name="hashblobqueue"></param> public static void SplitData(Stream inputstream, byte[] filehash, BlockingCollection <HashBlobPair> hashblobqueue) { // https://rsync.samba.org/tech_report/node3.html List <byte> newblob = new(); byte[] alphachksum = new byte[2]; byte[] betachksum = new byte[2]; SHA1 sha1filehasher = HashTools.GetSHA1Hasher(); SHA1 sha1blobhasher = HashTools.GetSHA1Hasher();; if (inputstream.Length != 0) { int readsize = 8_388_608; int rollwindowsize = 32; byte[] readin; byte[] shifted = new byte[2]; for (int i = 0; i < inputstream.Length; i += readsize) // read the file in larger chunks for efficiency { if (i + readsize <= inputstream.Length) // readsize or more bytes left to read { readin = new byte[readsize]; inputstream.Read(readin, 0, readsize); } else // < readsize bytes left to read { readin = new byte[inputstream.Length % readsize]; inputstream.Read(readin, 0, (int)(inputstream.Length % readsize)); } for (int j = 0; j < readin.Length; j++) // Byte by byte { newblob.Add(readin[j]); HashTools.ByteSum(alphachksum, newblob[^ 1]);
public IEnumerable <string> GetBackupsAndMetadataReferencesAsStrings(BackupSetReference bsname) { var bset = LoadBackupSet(bsname); foreach ((byte[] backupref, bool _) in bset.Backups) { yield return(HashTools.ByteArrayToHexViaLookup32(backupref)); foreach (byte[] reference in Dependencies.Blobs.GetAllBlobReferences(backupref, BlobLocation.BlobType.BackupRecord, false, false)) { yield return(HashTools.ByteArrayToHexViaLookup32(reference)); } } }
/// <summary> /// Loads the data from a blob, no special handling of multiblob references etc. /// </summary> /// <param name="blocation"></param> /// <param name="hash">Null for no verification</param> /// <returns></returns> private byte[] LoadBlob(BlobLocation blocation, byte[] hash, int retries = 0) { byte[] data = Dependencies.LoadBlob(blocation.EncryptedHash); byte[] datahash = HashTools.GetSHA1Hasher().ComputeHash(data); if (datahash.SequenceEqual(hash)) { return(data); } else if (retries > 0) { return(LoadBlob(blocation, hash, retries - 1)); } // NOTE: This hash check sometimes fails and throws the error, Issue #17 throw new Exception("Blob data did not match hash."); }
/// <summary> /// /// </summary> /// <param name="prefix"></param> /// <returns>Null if no matches, (true, null) for multiple matches, (false, hashstring) for exact match.</returns> public (bool multiplematches, byte[] singlematchhash)? HashByPrefix(string bsname, string prefix) { var bset = LoadBackupSet(bsname); // TODO: This implementation is pretty slow, could be improved with a better data structure like a trie or DAFSA // also if this becomes an issue, keep a s prefix = prefix.ToLower(); List <string> hashes = new List <string>(from backup in bset.Backups select HashTools.ByteArrayToHexViaLookup32(backup.hash)); List <string> matches = new List <string>(from h in hashes where h.ToLower().StartsWith(prefix.ToLower()) select h); if (matches.Count == 0) { return(null); } else if (matches.Count > 1) { return(true, null); } else { return(false, HashTools.HexStringToByteArray(matches[0])); } }
/// <summary> /// Given a path to a file, adds it to the file database and backs it up to the target path. /// </summary> /// <param name="db">The database containing catalogues files</param> /// <param name="targetPath">The path to copy the file to</param> /// <param name="file">The file to back up</param> private void CatalogueAndCopyNewFile(FileContext db, string targetPath, string file) { if (!Directory.Exists(targetPath.Replace(Path.GetFileName(file), ""))) { Directory.CreateDirectory(targetPath.Replace(Path.GetFileName(file), "")); //ensure the directory to backup to exists } File.Copy(file, targetPath, false); db.Files.Add(new ProcessedFile { FileName = Path.GetFileName(file), FilePath = file, BackupPath = targetPath, DateModified = File.GetLastWriteTime(file), FileHash = HashTools.HashFile(file) }); Console.WriteLine("Added new file " + Path.GetFileName(file)); }
public Task DeleteBlobAsync(byte[] hash, string fileId) { return(DeleteFileAsync(Path.Combine(BlobSaveDirectory, HashTools.ByteArrayToHexViaLookup32(hash)), fileId)); }
public Task <(byte[] encryptedHash, string fileId)> StoreBlobAsync(byte[] hash, byte[] data) { return(StoreFileAsync(Path.Combine(BlobSaveDirectory, HashTools.ByteArrayToHexViaLookup32(hash)), hash, data)); }
public Task <byte[]> LoadBlobAsync(byte[] hash) { return(LoadFileAsync(Path.Combine(BlobSaveDirectory, HashTools.ByteArrayToHexViaLookup32(hash)))); }
/// <summary> /// /// </summary> /// <param name="file"></param> /// <param name="hash"></param> /// <param name="data"></param> /// <returns>fileId</returns> private async Task <(byte[] encryptedHash, string fileId)> StoreFileAsync(string file, byte[] hash, byte[] data, bool preventEncryption = false) { async Task <GetUploadUrlResponse> GetUploadUrl(int attempts = 0) { if (AuthResp == null) { AuthResp = await AuthorizeAccount(); } Delay(); try { var urlresp = await AuthResp.apiUrl .AppendPathSegment("/b2api/v1/b2_get_upload_url") .WithHeaders(new { Authorization = AuthResp.authorizationToken }) .PostJsonAsync(new { bucketId = BucketId }) .ReceiveJson <GetUploadUrlResponse>().ConfigureAwait(false); SuccessfulTransmission(); return(urlresp); } catch (FlurlHttpException ex) { if (ex.Call.HttpStatus != null && ex.Call.HttpStatus == System.Net.HttpStatusCode.Unauthorized) { AuthResp = null; } else { // Other classes of errors may be congestion related so we increase the delay FailedTransmission(); } if (attempts < Retries) { return(await GetUploadUrl(attempts + 1).ConfigureAwait(false)); } throw; } } var hashFileId = await UploadData(); async Task <(byte[] encryptedHash, string fileId)> UploadData(int attempts = 0) { if (UploadUrlResp == null) { UploadUrlResp = await GetUploadUrl(); } Delay(); try { if (Encryptor != null && !preventEncryption) { data = Encryptor.EncryptBytes(data); hash = HashTools.GetSHA1Hasher().ComputeHash(data); } var filecontent = new ByteArrayContent(data); filecontent.Headers.Add("Content-Type", "application/octet-stream"); var uploadresp = await UploadUrlResp.uploadUrl .WithHeaders(new { Authorization = UploadUrlResp.authorizationToken, X_Bz_File_Name = file, Content_Length = data.Length, X_Bz_Content_Sha1 = HashTools.ByteArrayToHexViaLookup32(hash) }) .PostAsync(filecontent) .ReceiveJson <UploadResponse>().ConfigureAwait(false); SuccessfulTransmission(); return(hash, uploadresp.fileId); } catch (FlurlHttpException ex) { if (ex.Call.HttpStatus != null && ex.Call.HttpStatus == System.Net.HttpStatusCode.Unauthorized) { UploadUrlResp = null; } else { // Other classes of errors may be congestion related so we increase the delay FailedTransmission(); } if (attempts < Retries) { return(await UploadData(attempts + 1).ConfigureAwait(false)); } throw; } } return(hashFileId); }
private Task StoreFileAsync(string file, byte[] data, bool preventEncryption = false) => StoreFileAsync(file, HashTools.GetSHA1Hasher().ComputeHash(data), data, preventEncryption);
/// <summary> /// Chunks and saves data to blobstore. /// Operates on stream input, so Filestreams can be used and /// entire files need not be loaded into memory. /// If an error occurs (typically when reading from a stream /// representing a file), it is thrown to the caller. /// </summary> /// <param name="inputstream"></param> /// <param name="type"></param> /// <param name="filehash"></param> /// <param name="hashblobqueue"></param> public static void SplitData(Stream inputstream, byte[] filehash, BlockingCollection <HashBlobPair> hashblobqueue) { // https://rsync.samba.org/tech_report/node3.html List <byte> newblob = new List <byte>(); byte[] alphachksum = new byte[2]; byte[] betachksum = new byte[2]; SHA1 sha1filehasher = HashTools.GetSHA1Hasher(); SHA1 sha1blobhasher = HashTools.GetSHA1Hasher();; if (inputstream.Length != 0) { int readsize = 8_388_608; int rollwindowsize = 32; byte[] readin; byte[] shifted = new byte[2]; for (int i = 0; i < inputstream.Length; i += readsize) // read the file in larger chunks for efficiency { if (i + readsize <= inputstream.Length) // readsize or more bytes left to read { readin = new byte[readsize]; inputstream.Read(readin, 0, readsize); } else // < readsize bytes left to read { readin = new byte[inputstream.Length % readsize]; inputstream.Read(readin, 0, (int)(inputstream.Length % readsize)); } for (int j = 0; j < readin.Length; j++) // Byte by byte { newblob.Add(readin[j]); HashTools.ByteSum(alphachksum, newblob[newblob.Count - 1]); if (newblob.Count > rollwindowsize) { HashTools.ByteDifference(alphachksum, newblob[newblob.Count - rollwindowsize - 1]); shifted[0] = (byte)((newblob[newblob.Count - 1] << 5) & 0xFF); // rollwindowsize = 32 = 2^5 => 5 shifted[1] = (byte)((newblob[newblob.Count - 1] >> 3) & 0xFF); // 8-5 = 3 HashTools.BytesDifference(betachksum, shifted); } HashTools.BytesSum(betachksum, alphachksum); if (alphachksum[0] == 0xFF && betachksum[0] == 0xFF && betachksum[1] < 0x02) // (256*256*128)^-1 => expected value (/2) = ~4MB { byte[] blob = newblob.ToArray(); if (i + readsize >= inputstream.Length && j + 1 >= readin.Length) // Need to use TransformFinalBlock if at end of input { sha1filehasher.TransformFinalBlock(blob, 0, blob.Length); } else { sha1filehasher.TransformBlock(blob, 0, blob.Length, blob, 0); } hashblobqueue.Add(new HashBlobPair(sha1blobhasher.ComputeHash(blob), blob)); newblob = new List <byte>(); Array.Clear(alphachksum, 0, 2); Array.Clear(betachksum, 0, 2); } } } if (newblob.Count != 0) // Create blob from remaining bytes { byte[] blob = newblob.ToArray(); sha1filehasher.TransformFinalBlock(blob, 0, blob.Length); hashblobqueue.Add(new HashBlobPair(sha1blobhasher.ComputeHash(blob), blob)); } } else { byte[] blob = new byte[0]; sha1filehasher.TransformFinalBlock(blob, 0, blob.Length); hashblobqueue.Add(new HashBlobPair(sha1blobhasher.ComputeHash(blob), blob)); } Array.Copy(sha1filehasher.Hash, filehash, sha1filehasher.Hash.Length); hashblobqueue.CompleteAdding(); }
public (string, BackupRecord) GetBackupHashAndRecord(string bsname, int offset = 0) { var bset = LoadBackupSet(bsname); return(GetBackupHashAndRecord(bsname, HashTools.ByteArrayToHexViaLookup32(bset.Backups[bset.Backups.Count - 1].hash).ToLower(), offset)); }