public void FinalizeBackupAddition(BackupSetReference bsname, byte[] backuphash, byte[] mtreehash, HashTreeNode mtreereferences)
{
BlobLocation backupblocation = GetBlobLocation(backuphash);
int? backupRefCount = backupblocation.GetBSetReferenceCount(bsname);
if (!backupRefCount.HasValue || backupRefCount == 0)
{
BlobLocation mtreeblocation = GetBlobLocation(mtreehash);
int? mtreeRefCount = mtreeblocation.GetBSetReferenceCount(bsname);
if (!mtreeRefCount.HasValue || mtreeRefCount == 0)
{
ISkippableChildrenIterator <byte[]> childReferences = mtreereferences.GetChildIterator();
foreach (var blobhash in childReferences)
{
BlobLocation blocation = GetBlobLocation(blobhash);
int? refCount = blocation.GetBSetReferenceCount(bsname);
if (refCount.HasValue && refCount > 0) // This was already stored
{
childReferences.SkipChildrenOfCurrent();
}
else if (blocation.BlockHashes != null)
{
foreach (var mbref in blocation.BlockHashes)
{
IncrementReferenceCountNoRecurse(bsname, mbref, 1);
}
}
IncrementReferenceCountNoRecurse(bsname, blocation, blobhash, 1);
}
IncrementReferenceCountNoRecurse(bsname, mtreeblocation, mtreehash, 1);
}
IncrementReferenceCountNoRecurse(bsname, backupblocation, backuphash, 1);
}
}
public void DecrementReferenceCount(BackupSetReference backupsetname, byte[] blobhash, BlobLocation.BlobType blobtype,
bool includefiles)
{
BlobLocation rootBlobLocation = GetBlobLocation(blobhash);
if (rootBlobLocation.GetBSetReferenceCount(backupsetname) == 1) // To be deleted?
{
IBlobReferenceIterator blobReferences = GetAllBlobReferences(blobhash, blobtype, includefiles, false);
foreach (var reference in blobReferences)
{
BlobLocation blocation = GetBlobLocation(reference);
// When we finish iterating over the children, decrement this blob
blobReferences.PostOrderAction(() => IncrementReferenceCountNoRecurse(backupsetname, blocation, reference, -1));
try
{
if (blocation.GetBSetReferenceCount(backupsetname) != 1) // Not to be deleted?
{
// Dont need to decrement child references if this wont be deleted
blobReferences.SkipChildrenOfCurrent();
}
}
catch (KeyNotFoundException)
{
throw;
}
}
}
IncrementReferenceCountNoRecurse(backupsetname, rootBlobLocation, blobhash, -1); // must delete parent last so parent can be loaded/used in GetAllBlobReferences()
}
public void FinalizeBackupAddition(string bsname, byte[] backuphash, byte[] mtreehash, HashTreeNode mtreereferences)
{
BlobLocation backupblocation = GetBlobLocation(backuphash);
if (!backupblocation.BSetReferenceCounts.ContainsKey(bsname) || backupblocation.BSetReferenceCounts[bsname] == 0)
{
BlobLocation mtreeblocation = GetBlobLocation(mtreehash);
if (!backupblocation.BSetReferenceCounts.ContainsKey(bsname) || mtreeblocation.BSetReferenceCounts[bsname] == 0)
{
ISkippableChildrenIterator <byte[]> childReferences = mtreereferences.GetChildIterator();
foreach (var blobhash in childReferences)
{
BlobLocation blocation = GetBlobLocation(blobhash);
if (backupblocation.BSetReferenceCounts.ContainsKey(bsname) && blocation.BSetReferenceCounts[bsname] > 0) // This was already stored
{
childReferences.SkipChildren();
}
else if (blocation.BlockHashes != null)
{
foreach (var mbref in blocation.BlockHashes)
{
IncrementReferenceCountNoRecurse(bsname, mbref, 1);
}
}
IncrementReferenceCountNoRecurse(bsname, blocation, blobhash, 1);
}
IncrementReferenceCountNoRecurse(bsname, mtreeblocation, mtreehash, 1);
}
IncrementReferenceCountNoRecurse(bsname, backupblocation, backuphash, 1);
}
}
public BlobLocation GetBlobLocation(byte[] hash)
{
BlobLocation blocation = IndexStore.GetRecord(hash);
if (blocation == null)
{
throw new KeyNotFoundException("The given hash does not exist in the Blob Index.");
}
return(blocation);
}
public bool ContainsHash(string backupset, byte[] hash)
{
BlobLocation blocation = IndexStore.GetRecord(hash);
if (blocation != null)
{
return(blocation.BSetReferenceCounts.ContainsKey(backupset));
}
return(false);
}
/// <summary>
/// Adds a hash and corresponding BackupLocation to the Index.
/// Does not modify reference counts.
/// </summary>
/// <param name="hash"></param>
/// <returns>
/// Returns the existing location if we already have the hash stored. Null if we used
/// the parameter blocation and need to save the corresponding blob data.
/// </returns>
private (BlobLocation existinglocation, bool datastored)? AddHash(byte[] hash, BlobLocation blocation)
{
// Adds a hash and Blob Location to the BlockHashStore
BlobLocation existingblocation = IndexStore.AddOrFind(hash, blocation);
if (existingblocation == null)
{
return(null);
}
return(existingblocation, existingblocation.TotalNonShallowReferenceCount > 0);
}
/// <summary>
/// Loads the data from a blob, no special handling of multiblob references etc.
/// </summary>
/// <param name="blocation"></param>
/// <param name="hash">Null for no verification</param>
/// <returns></returns>
private byte[] LoadBlob(BlobLocation blocation, byte[] hash, int retries = 0)
{
byte[] data = Dependencies.LoadBlob(blocation.EncryptedHash);
byte[] datahash = HashTools.GetSHA1Hasher().ComputeHash(data);
if (datahash.SequenceEqual(hash))
{
return(data);
}
else if (retries > 0)
{
return(LoadBlob(blocation, hash, retries - 1));
}
// NOTE: This hash check sometimes fails and throws the error, Issue #17
throw new Exception("Blob data did not match hash.");
}
public byte[] RetrieveData(byte[] filehash, BlobLocation blobLocation)
{
if (blobLocation.BlockHashes != null) // File is comprised of multiple blobs
{
MemoryStream outstream = new();
foreach (var hash in blobLocation.BlockHashes)
{
BlobLocation blobloc = GetBlobLocation(hash);
outstream.Write(LoadBlob(blobloc, hash), 0, blobloc.ByteLength);
}
byte[] filedata = outstream.ToArray();
outstream.Close();
return(filedata);
}
else // file is single blob
{
return(LoadBlob(blobLocation, filehash));
}
}
private void IncrementReferenceCountNoRecurse(string backupset, BlobLocation blocation, byte[] blobhash, int amount)
{
bool originallyshallow = blocation.TotalNonShallowReferenceCount == 0;
if (blocation.BSetReferenceCounts.ContainsKey(backupset))
{
blocation.BSetReferenceCounts[backupset] += amount;
}
else
{
blocation.BSetReferenceCounts[backupset] = amount;
}
if (blocation.BSetReferenceCounts[backupset] == 0)
{
blocation.BSetReferenceCounts.Remove(backupset);
}
else if (blocation.BSetReferenceCounts[backupset] < 0)
{
throw new Exception("Negative reference count in blobstore");
}
if (blocation.BlockHashes == null) // Can't delete from disk if this is a multiblock reference (does not directly store data on disk)
{
if (blocation.TotalNonShallowReferenceCount == 0)
{
if (!originallyshallow)
{
try
{
Dependencies.DeleteBlob(blocation.EncryptedHash, blocation.RelativeFilePath);
}
catch (Exception)
{
throw new Exception("Error deleting unreferenced file.");
}
}
}
}
if (blocation.TotalReferenceCount == 0)
{
IndexStore.Remove(blobhash);
}
}
private void IncrementReferenceCountNoRecurse(BackupSetReference backupset, BlobLocation blocation, byte[] blobhash, int amount)
{
bool originallyshallow = blocation.TotalNonShallowReferenceCount == 0;
int? refCount = blocation.GetBSetReferenceCount(backupset);
int newRefCount = refCount.GetValueOrDefault(0) + amount;
blocation.SetBSetReferenceCount(backupset, newRefCount);
if (newRefCount == 0)
{
blocation.RemoveBSetReference(backupset);
}
else if (newRefCount < 0)
{
throw new Exception("Negative reference count in blobstore");
}
if (blocation.BlockHashes == null) // Can't delete from disk if this is a multiblock reference (does not directly store data on disk)
{
if (blocation.TotalNonShallowReferenceCount == 0)
{
if (!originallyshallow)
{
try
{
if (blocation.EncryptedHash == null)
{
throw new Exception("Hash should not be null");
}
Dependencies.DeleteBlob(blocation.EncryptedHash, blocation.RelativeFilePath);
}
catch (Exception e)
{
throw new Exception("Error deleting unreferenced file.", e);
}
}
}
}
if (blocation.TotalReferenceCount == 0)
{
IndexStore.Remove(blobhash);
}
}
public void FinalizeBlobAddition(BackupSetReference bsname, byte[] blobhash, BlobLocation.BlobType blobType)
{
// Handle root blob
BlobLocation rootblocation = GetBlobLocation(blobhash);
if (rootblocation.TotalReferenceCount == 0)
{
IBlobReferenceIterator blobReferences = GetAllBlobReferences(blobhash, blobType, true, false);
// Loop through children
foreach (byte[] reference in blobReferences)
{
BlobLocation blocation = GetBlobLocation(reference);
if (blocation.TotalReferenceCount > 0) // This was already stored
{
blobReferences.SkipChildrenOfCurrent();
}
IncrementReferenceCountNoRecurse(bsname, blocation, blobhash, 1);
}
}
// Increment root blob
IncrementReferenceCountNoRecurse(bsname, rootblocation, blobhash, 1);
}
public byte[] RetrieveData(byte[] filehash)
{
BlobLocation blobbl = GetBlobLocation(filehash);
return(RetrieveData(filehash, blobbl));
}
private BlobLocation AddBlob(BackupSetReference backupset, HashBlobPair blob, List <byte[]>?blockreferences, bool shallow = false)
{
// We navigate down
// Where we will put the blob data if we dont already have it stored
BlobLocation posblocation;
if (shallow)
{
posblocation = new BlobLocation(blockreferences);
}
else
{
if (blockreferences == null)
{
if (blob.Block == null)
{
throw new Exception("Block can only be null in multirefernce blob");
}
posblocation = new BlobLocation(null, "", blob.Block.Length);
}
else
{
posblocation = new BlobLocation(blockreferences);
}
}
// Where the data is already stored if it exists
(BlobLocation bloc, bool datastored)? existingblocstored;
lock (this)
{
// Have we already stored this?
existingblocstored = AddHash(blob.Hash, posblocation);
}
if (existingblocstored == null) // ExistBloc == null means posbloc was just added
{
if (!shallow)
{
if (blockreferences == null)
{
if (blob.Block == null)
{
throw new Exception("Block can only be null in multirefernce blob");
}
(posblocation.EncryptedHash, posblocation.RelativeFilePath) = WriteBlob(blob.Hash, blob.Block);
}
}
else
{
posblocation.RelativeFilePath = "";
posblocation.EncryptedHash = blob.Hash;
}
// Dont change reference counts until finalization
// IncrementReferenceCountNoRecurse(backupset, posblocation, blob.Hash, 1);
return(posblocation);
}
else // Existbloc already stored at dst
{
(BlobLocation existingbloc, bool datastored) = existingblocstored.Value;
// Is the data not already stored in the blobstore (are all references shallow thus far)?
if (existingbloc.BlockHashes == null)
{
if (!datastored)
{
// Data is not already stored
// Dont save if we are writing a bloblistcache
if (!backupset.BlobListCache)
{
// If we are saving to a cache and the bloblist cache indicates the destination has the data
// Then dont store, Else save
//BackupSetReference blobListCacheReference = backupset with { BlobListCache = true };
if (!(backupset.Cache &&
existingbloc.GetBSetReferenceCount(backupset).HasValue))
{
if (blob.Block == null)
{
throw new Exception("Block can only be null in multirefernce blob");
}
(existingbloc.EncryptedHash, existingbloc.RelativeFilePath) = WriteBlob(blob.Hash, blob.Block);
}
}
}
}
// Dont change reference counts until finalization
// IncrementReferenceCountNoRecurse(backupset, existingbloc, blob.Hash, 1);
return(existingbloc);
}
}
/// <summary>
///
/// </summary>
/// <param name="dst"></param>
/// <param name="blobhash"></param>
/// <returns>True Blob exists in destination</returns>
private (BlobLocation bloc, byte[]? blob) TransferBlobNoReferences(BlobStore dst, BackupSetReference dstbackupset,
byte[] blobhash, BlobLocation blocation)
{
if (blocation.BlockHashes == null)
{
byte[] blob = LoadBlob(blocation, blobhash);
return(dst.AddBlob(dstbackupset, new HashBlobPair(blobhash, blob)), blob);
}
else
{
return(dst.AddMultiBlobReferenceBlob(dstbackupset, blobhash, blocation.BlockHashes), null);
}
}
private BlobLocation AddBlob(string backupset, HashBlobPair blob, List <byte[]> blockreferences, bool shallow = false)
{
// We navigate down
// Where we will put the blob data if we dont already have it stored
BlobLocation posblocation;
if (shallow)
{
posblocation = new BlobLocation(blockreferences);
}
else
{
if (blockreferences == null)
{
posblocation = new BlobLocation(null, "", blob.Block.Length);
}
else
{
posblocation = new BlobLocation(blockreferences);
}
}
// Where the data is already stored if it exists
(BlobLocation bloc, bool datastored)? existingblocstored;
lock (this)
{
// Have we already stored this?
existingblocstored = AddHash(blob.Hash, posblocation);
}
if (existingblocstored == null) // ExistBloc == null means posbloc was just added
{
if (!shallow)
{
if (blockreferences == null)
{
(posblocation.EncryptedHash, posblocation.RelativeFilePath) = WriteBlob(blob.Hash, blob.Block);
}
}
else
{
posblocation.RelativeFilePath = "";
posblocation.EncryptedHash = blob.Hash;
}
// Dont change reference counts until finalization
// IncrementReferenceCountNoRecurse(backupset, posblocation, blob.Hash, 1);
return(posblocation);
}
else // Existbloc already stored at dst
{
(BlobLocation existingbloc, bool datastored) = existingblocstored.Value;
// Is the data not already stored in the blobstore (are all references shallow thus far)?
if (existingbloc.BlockHashes == null)
{
if (!datastored)
{
// Data is not already stored
// Dont save if we are writing a bloblistcache
if (!backupset.EndsWith(Core.BlobListCacheSuffix))
{
// If we are saving to a cache and the bloblist cache indicates the destination has the data
// Then dont store, Else save
if (!(backupset.EndsWith(Core.CacheSuffix) &&
existingbloc.BSetReferenceCounts.ContainsKey(backupset.Substring(0,
backupset.Length - Core.CacheSuffix.Length) + Core.BlobListCacheSuffix)))
{
(existingbloc.EncryptedHash, existingbloc.RelativeFilePath) = WriteBlob(blob.Hash, blob.Block);
}
}
}
}
// Dont change reference counts until finalization
// IncrementReferenceCountNoRecurse(backupset, existingbloc, blob.Hash, 1);
return(existingbloc);
}
}