public void FinalizeBackupAddition(BackupSetReference bsname, byte[] backuphash, byte[] mtreehash, HashTreeNode mtreereferences) { BlobLocation backupblocation = GetBlobLocation(backuphash); int? backupRefCount = backupblocation.GetBSetReferenceCount(bsname); if (!backupRefCount.HasValue || backupRefCount == 0) { BlobLocation mtreeblocation = GetBlobLocation(mtreehash); int? mtreeRefCount = mtreeblocation.GetBSetReferenceCount(bsname); if (!mtreeRefCount.HasValue || mtreeRefCount == 0) { ISkippableChildrenIterator <byte[]> childReferences = mtreereferences.GetChildIterator(); foreach (var blobhash in childReferences) { BlobLocation blocation = GetBlobLocation(blobhash); int? refCount = blocation.GetBSetReferenceCount(bsname); if (refCount.HasValue && refCount > 0) // This was already stored { childReferences.SkipChildrenOfCurrent(); } else if (blocation.BlockHashes != null) { foreach (var mbref in blocation.BlockHashes) { IncrementReferenceCountNoRecurse(bsname, mbref, 1); } } IncrementReferenceCountNoRecurse(bsname, blocation, blobhash, 1); } IncrementReferenceCountNoRecurse(bsname, mtreeblocation, mtreehash, 1); } IncrementReferenceCountNoRecurse(bsname, backupblocation, backuphash, 1); } }
public void DecrementReferenceCount(BackupSetReference backupsetname, byte[] blobhash, BlobLocation.BlobType blobtype, bool includefiles) { BlobLocation rootBlobLocation = GetBlobLocation(blobhash); if (rootBlobLocation.GetBSetReferenceCount(backupsetname) == 1) // To be deleted? { IBlobReferenceIterator blobReferences = GetAllBlobReferences(blobhash, blobtype, includefiles, false); foreach (var reference in blobReferences) { BlobLocation blocation = GetBlobLocation(reference); // When we finish iterating over the children, decrement this blob blobReferences.PostOrderAction(() => IncrementReferenceCountNoRecurse(backupsetname, blocation, reference, -1)); try { if (blocation.GetBSetReferenceCount(backupsetname) != 1) // Not to be deleted? { // Dont need to decrement child references if this wont be deleted blobReferences.SkipChildrenOfCurrent(); } } catch (KeyNotFoundException) { throw; } } } IncrementReferenceCountNoRecurse(backupsetname, rootBlobLocation, blobhash, -1); // must delete parent last so parent can be loaded/used in GetAllBlobReferences() }
public void FinalizeBackupAddition(string bsname, byte[] backuphash, byte[] mtreehash, HashTreeNode mtreereferences) { BlobLocation backupblocation = GetBlobLocation(backuphash); if (!backupblocation.BSetReferenceCounts.ContainsKey(bsname) || backupblocation.BSetReferenceCounts[bsname] == 0) { BlobLocation mtreeblocation = GetBlobLocation(mtreehash); if (!backupblocation.BSetReferenceCounts.ContainsKey(bsname) || mtreeblocation.BSetReferenceCounts[bsname] == 0) { ISkippableChildrenIterator <byte[]> childReferences = mtreereferences.GetChildIterator(); foreach (var blobhash in childReferences) { BlobLocation blocation = GetBlobLocation(blobhash); if (backupblocation.BSetReferenceCounts.ContainsKey(bsname) && blocation.BSetReferenceCounts[bsname] > 0) // This was already stored { childReferences.SkipChildren(); } else if (blocation.BlockHashes != null) { foreach (var mbref in blocation.BlockHashes) { IncrementReferenceCountNoRecurse(bsname, mbref, 1); } } IncrementReferenceCountNoRecurse(bsname, blocation, blobhash, 1); } IncrementReferenceCountNoRecurse(bsname, mtreeblocation, mtreehash, 1); } IncrementReferenceCountNoRecurse(bsname, backupblocation, backuphash, 1); } }
public BlobLocation GetBlobLocation(byte[] hash) { BlobLocation blocation = IndexStore.GetRecord(hash); if (blocation == null) { throw new KeyNotFoundException("The given hash does not exist in the Blob Index."); } return(blocation); }
public bool ContainsHash(string backupset, byte[] hash) { BlobLocation blocation = IndexStore.GetRecord(hash); if (blocation != null) { return(blocation.BSetReferenceCounts.ContainsKey(backupset)); } return(false); }
/// <summary> /// Adds a hash and corresponding BackupLocation to the Index. /// Does not modify reference counts. /// </summary> /// <param name="hash"></param> /// <returns> /// Returns the existing location if we already have the hash stored. Null if we used /// the parameter blocation and need to save the corresponding blob data. /// </returns> private (BlobLocation existinglocation, bool datastored)? AddHash(byte[] hash, BlobLocation blocation) { // Adds a hash and Blob Location to the BlockHashStore BlobLocation existingblocation = IndexStore.AddOrFind(hash, blocation); if (existingblocation == null) { return(null); } return(existingblocation, existingblocation.TotalNonShallowReferenceCount > 0); }
/// <summary> /// Loads the data from a blob, no special handling of multiblob references etc. /// </summary> /// <param name="blocation"></param> /// <param name="hash">Null for no verification</param> /// <returns></returns> private byte[] LoadBlob(BlobLocation blocation, byte[] hash, int retries = 0) { byte[] data = Dependencies.LoadBlob(blocation.EncryptedHash); byte[] datahash = HashTools.GetSHA1Hasher().ComputeHash(data); if (datahash.SequenceEqual(hash)) { return(data); } else if (retries > 0) { return(LoadBlob(blocation, hash, retries - 1)); } // NOTE: This hash check sometimes fails and throws the error, Issue #17 throw new Exception("Blob data did not match hash."); }
public byte[] RetrieveData(byte[] filehash, BlobLocation blobLocation) { if (blobLocation.BlockHashes != null) // File is comprised of multiple blobs { MemoryStream outstream = new(); foreach (var hash in blobLocation.BlockHashes) { BlobLocation blobloc = GetBlobLocation(hash); outstream.Write(LoadBlob(blobloc, hash), 0, blobloc.ByteLength); } byte[] filedata = outstream.ToArray(); outstream.Close(); return(filedata); } else // file is single blob { return(LoadBlob(blobLocation, filehash)); } }
private void IncrementReferenceCountNoRecurse(string backupset, BlobLocation blocation, byte[] blobhash, int amount) { bool originallyshallow = blocation.TotalNonShallowReferenceCount == 0; if (blocation.BSetReferenceCounts.ContainsKey(backupset)) { blocation.BSetReferenceCounts[backupset] += amount; } else { blocation.BSetReferenceCounts[backupset] = amount; } if (blocation.BSetReferenceCounts[backupset] == 0) { blocation.BSetReferenceCounts.Remove(backupset); } else if (blocation.BSetReferenceCounts[backupset] < 0) { throw new Exception("Negative reference count in blobstore"); } if (blocation.BlockHashes == null) // Can't delete from disk if this is a multiblock reference (does not directly store data on disk) { if (blocation.TotalNonShallowReferenceCount == 0) { if (!originallyshallow) { try { Dependencies.DeleteBlob(blocation.EncryptedHash, blocation.RelativeFilePath); } catch (Exception) { throw new Exception("Error deleting unreferenced file."); } } } } if (blocation.TotalReferenceCount == 0) { IndexStore.Remove(blobhash); } }
private void IncrementReferenceCountNoRecurse(BackupSetReference backupset, BlobLocation blocation, byte[] blobhash, int amount) { bool originallyshallow = blocation.TotalNonShallowReferenceCount == 0; int? refCount = blocation.GetBSetReferenceCount(backupset); int newRefCount = refCount.GetValueOrDefault(0) + amount; blocation.SetBSetReferenceCount(backupset, newRefCount); if (newRefCount == 0) { blocation.RemoveBSetReference(backupset); } else if (newRefCount < 0) { throw new Exception("Negative reference count in blobstore"); } if (blocation.BlockHashes == null) // Can't delete from disk if this is a multiblock reference (does not directly store data on disk) { if (blocation.TotalNonShallowReferenceCount == 0) { if (!originallyshallow) { try { if (blocation.EncryptedHash == null) { throw new Exception("Hash should not be null"); } Dependencies.DeleteBlob(blocation.EncryptedHash, blocation.RelativeFilePath); } catch (Exception e) { throw new Exception("Error deleting unreferenced file.", e); } } } } if (blocation.TotalReferenceCount == 0) { IndexStore.Remove(blobhash); } }
public void FinalizeBlobAddition(BackupSetReference bsname, byte[] blobhash, BlobLocation.BlobType blobType) { // Handle root blob BlobLocation rootblocation = GetBlobLocation(blobhash); if (rootblocation.TotalReferenceCount == 0) { IBlobReferenceIterator blobReferences = GetAllBlobReferences(blobhash, blobType, true, false); // Loop through children foreach (byte[] reference in blobReferences) { BlobLocation blocation = GetBlobLocation(reference); if (blocation.TotalReferenceCount > 0) // This was already stored { blobReferences.SkipChildrenOfCurrent(); } IncrementReferenceCountNoRecurse(bsname, blocation, blobhash, 1); } } // Increment root blob IncrementReferenceCountNoRecurse(bsname, rootblocation, blobhash, 1); }
public byte[] RetrieveData(byte[] filehash) { BlobLocation blobbl = GetBlobLocation(filehash); return(RetrieveData(filehash, blobbl)); }
private BlobLocation AddBlob(BackupSetReference backupset, HashBlobPair blob, List <byte[]>?blockreferences, bool shallow = false) { // We navigate down // Where we will put the blob data if we dont already have it stored BlobLocation posblocation; if (shallow) { posblocation = new BlobLocation(blockreferences); } else { if (blockreferences == null) { if (blob.Block == null) { throw new Exception("Block can only be null in multirefernce blob"); } posblocation = new BlobLocation(null, "", blob.Block.Length); } else { posblocation = new BlobLocation(blockreferences); } } // Where the data is already stored if it exists (BlobLocation bloc, bool datastored)? existingblocstored; lock (this) { // Have we already stored this? existingblocstored = AddHash(blob.Hash, posblocation); } if (existingblocstored == null) // ExistBloc == null means posbloc was just added { if (!shallow) { if (blockreferences == null) { if (blob.Block == null) { throw new Exception("Block can only be null in multirefernce blob"); } (posblocation.EncryptedHash, posblocation.RelativeFilePath) = WriteBlob(blob.Hash, blob.Block); } } else { posblocation.RelativeFilePath = ""; posblocation.EncryptedHash = blob.Hash; } // Dont change reference counts until finalization // IncrementReferenceCountNoRecurse(backupset, posblocation, blob.Hash, 1); return(posblocation); } else // Existbloc already stored at dst { (BlobLocation existingbloc, bool datastored) = existingblocstored.Value; // Is the data not already stored in the blobstore (are all references shallow thus far)? if (existingbloc.BlockHashes == null) { if (!datastored) { // Data is not already stored // Dont save if we are writing a bloblistcache if (!backupset.BlobListCache) { // If we are saving to a cache and the bloblist cache indicates the destination has the data // Then dont store, Else save //BackupSetReference blobListCacheReference = backupset with { BlobListCache = true }; if (!(backupset.Cache && existingbloc.GetBSetReferenceCount(backupset).HasValue)) { if (blob.Block == null) { throw new Exception("Block can only be null in multirefernce blob"); } (existingbloc.EncryptedHash, existingbloc.RelativeFilePath) = WriteBlob(blob.Hash, blob.Block); } } } } // Dont change reference counts until finalization // IncrementReferenceCountNoRecurse(backupset, existingbloc, blob.Hash, 1); return(existingbloc); } }
/// <summary> /// /// </summary> /// <param name="dst"></param> /// <param name="blobhash"></param> /// <returns>True Blob exists in destination</returns> private (BlobLocation bloc, byte[]? blob) TransferBlobNoReferences(BlobStore dst, BackupSetReference dstbackupset, byte[] blobhash, BlobLocation blocation) { if (blocation.BlockHashes == null) { byte[] blob = LoadBlob(blocation, blobhash); return(dst.AddBlob(dstbackupset, new HashBlobPair(blobhash, blob)), blob); } else { return(dst.AddMultiBlobReferenceBlob(dstbackupset, blobhash, blocation.BlockHashes), null); } }
private BlobLocation AddBlob(string backupset, HashBlobPair blob, List <byte[]> blockreferences, bool shallow = false) { // We navigate down // Where we will put the blob data if we dont already have it stored BlobLocation posblocation; if (shallow) { posblocation = new BlobLocation(blockreferences); } else { if (blockreferences == null) { posblocation = new BlobLocation(null, "", blob.Block.Length); } else { posblocation = new BlobLocation(blockreferences); } } // Where the data is already stored if it exists (BlobLocation bloc, bool datastored)? existingblocstored; lock (this) { // Have we already stored this? existingblocstored = AddHash(blob.Hash, posblocation); } if (existingblocstored == null) // ExistBloc == null means posbloc was just added { if (!shallow) { if (blockreferences == null) { (posblocation.EncryptedHash, posblocation.RelativeFilePath) = WriteBlob(blob.Hash, blob.Block); } } else { posblocation.RelativeFilePath = ""; posblocation.EncryptedHash = blob.Hash; } // Dont change reference counts until finalization // IncrementReferenceCountNoRecurse(backupset, posblocation, blob.Hash, 1); return(posblocation); } else // Existbloc already stored at dst { (BlobLocation existingbloc, bool datastored) = existingblocstored.Value; // Is the data not already stored in the blobstore (are all references shallow thus far)? if (existingbloc.BlockHashes == null) { if (!datastored) { // Data is not already stored // Dont save if we are writing a bloblistcache if (!backupset.EndsWith(Core.BlobListCacheSuffix)) { // If we are saving to a cache and the bloblist cache indicates the destination has the data // Then dont store, Else save if (!(backupset.EndsWith(Core.CacheSuffix) && existingbloc.BSetReferenceCounts.ContainsKey(backupset.Substring(0, backupset.Length - Core.CacheSuffix.Length) + Core.BlobListCacheSuffix))) { (existingbloc.EncryptedHash, existingbloc.RelativeFilePath) = WriteBlob(blob.Hash, blob.Block); } } } } // Dont change reference counts until finalization // IncrementReferenceCountNoRecurse(backupset, existingbloc, blob.Hash, 1); return(existingbloc); } }