Beispiel #1
0
        /// <summary>
        /// Gets the latest BackupRecord in this BackupStore.
        /// Returns null if no backups.
        /// </summary>
        /// <param name="bsname"></param>
        /// <returns></returns>
        public BackupRecord GetBackupRecord(BackupSetReference bsname)
        {
            var bset = LoadBackupSet(bsname);

            if (bset.Backups.Count > 0)
            {
                return(GetBackupRecord(bsname, bset.Backups[^ 1].hash));
Beispiel #2
0
        public BackupBrowser(string backupset, string?backuphash, BackupCore.Core bcore, int backupdst = 0)
        {
            ContinueLoop = true;
            BCore        = bcore;
            BackupDst    = backupdst;
            BackupSetReference backupSetReference = new BackupSetReference("test", false, false, false);

            if (!BCore.DestinationAvailable)
            {
                backupSetReference = backupSetReference with {
                    Cache = true
                };
            }
            (string hash, BackupCore.BackupRecord record)targetbackuphashandrecord;
            if (backuphash == null)
            {
                targetbackuphashandrecord = BCore.DefaultDstDependencies[BackupDst].Backups.GetBackupHashAndRecord(backupSetReference);
            }
            else
            {
                targetbackuphashandrecord = BCore.DefaultDstDependencies[BackupDst].Backups.GetBackupHashAndRecord(backupSetReference, backuphash, 0);
            }
            BackupHash = targetbackuphashandrecord.hash;
            BackupSet  = backupset;
            BackupCore.BackupRecord backuprecord = targetbackuphashandrecord.record;
            BackupTree  = BackupCore.MetadataNode.Load(BCore.DefaultDstDependencies[BackupDst].Blobs, backuprecord.MetadataTreeHash);
            CurrentNode = BackupTree;
        }
Beispiel #3
0
        public void FinalizeBackupAddition(BackupSetReference bsname, byte[] backuphash, byte[] mtreehash, HashTreeNode mtreereferences)
        {
            BlobLocation backupblocation = GetBlobLocation(backuphash);
            int?         backupRefCount  = backupblocation.GetBSetReferenceCount(bsname);

            if (!backupRefCount.HasValue || backupRefCount == 0)
            {
                BlobLocation mtreeblocation = GetBlobLocation(mtreehash);
                int?         mtreeRefCount  = mtreeblocation.GetBSetReferenceCount(bsname);
                if (!mtreeRefCount.HasValue || mtreeRefCount == 0)
                {
                    ISkippableChildrenIterator <byte[]> childReferences = mtreereferences.GetChildIterator();
                    foreach (var blobhash in childReferences)
                    {
                        BlobLocation blocation = GetBlobLocation(blobhash);
                        int?         refCount  = blocation.GetBSetReferenceCount(bsname);
                        if (refCount.HasValue && refCount > 0) // This was already stored
                        {
                            childReferences.SkipChildrenOfCurrent();
                        }
                        else if (blocation.BlockHashes != null)
                        {
                            foreach (var mbref in blocation.BlockHashes)
                            {
                                IncrementReferenceCountNoRecurse(bsname, mbref, 1);
                            }
                        }
                        IncrementReferenceCountNoRecurse(bsname, blocation, blobhash, 1);
                    }

                    IncrementReferenceCountNoRecurse(bsname, mtreeblocation, mtreehash, 1);
                }
                IncrementReferenceCountNoRecurse(bsname, backupblocation, backuphash, 1);
            }
        }
Beispiel #4
0
        public void DecrementReferenceCount(BackupSetReference backupsetname, byte[] blobhash, BlobLocation.BlobType blobtype,
                                            bool includefiles)
        {
            BlobLocation rootBlobLocation = GetBlobLocation(blobhash);

            if (rootBlobLocation.GetBSetReferenceCount(backupsetname) == 1) // To be deleted?
            {
                IBlobReferenceIterator blobReferences = GetAllBlobReferences(blobhash, blobtype, includefiles, false);
                foreach (var reference in blobReferences)
                {
                    BlobLocation blocation = GetBlobLocation(reference);

                    // When we finish iterating over the children, decrement this blob
                    blobReferences.PostOrderAction(() => IncrementReferenceCountNoRecurse(backupsetname, blocation, reference, -1));
                    try
                    {
                        if (blocation.GetBSetReferenceCount(backupsetname) != 1) // Not to be deleted?
                        {
                            // Dont need to decrement child references if this wont be deleted
                            blobReferences.SkipChildrenOfCurrent();
                        }
                    }
                    catch (KeyNotFoundException)
                    {
                        throw;
                    }
                }
            }
            IncrementReferenceCountNoRecurse(backupsetname, rootBlobLocation, blobhash, -1); // must delete parent last so parent can be loaded/used in GetAllBlobReferences()
        }
Beispiel #5
0
 public int?GetBSetReferenceCount(BackupSetReference backupSet)
 {
     if (BSetReferenceCounts.TryGetValue(new BackupSetKey(backupSet.BackupSetName, backupSet.Shallow, backupSet.BlobListCache), out int refCount))
     {
         return(refCount);
     }
     return(null);
 }
Beispiel #6
0
        public bool ContainsHash(BackupSetReference backupset, byte[] hash)
        {
            BlobLocation?blocation = IndexStore.GetRecord(hash);

            if (blocation != null)
            {
                return(blocation.GetBSetReferenceCount(backupset).HasValue);
            }
            return(false);
        }
Beispiel #7
0
 // TODO: should we just have to iteratively remove each backup in the bset?
 public void RemoveAllBackupSetReferences(BackupSetReference bsname)
 {
     foreach (KeyValuePair <byte[], BlobLocation> hashblob in IndexStore)
     {
         int?refCount = hashblob.Value.GetBSetReferenceCount(bsname);
         if (refCount != null)
         {
             IncrementReferenceCountNoRecurse(bsname, hashblob.Key, -refCount.Value);
         }
     }
 }
Beispiel #8
0
        public IEnumerable <string> GetBackupsAndMetadataReferencesAsStrings(BackupSetReference bsname)
        {
            var bset = LoadBackupSet(bsname);

            foreach ((byte[] backupref, bool _) in bset.Backups)
            {
                yield return(HashTools.ByteArrayToHexViaLookup32(backupref));

                foreach (byte[] reference in Dependencies.Blobs.GetAllBlobReferences(backupref,
                                                                                     BlobLocation.BlobType.BackupRecord, false, false))
                {
                    yield return(HashTools.ByteArrayToHexViaLookup32(reference));
                }
            }
        }
Beispiel #9
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="bsname"></param>
        /// <param name="message"></param>
        /// <param name="metadatatreehash"></param>
        /// <param name="shallow"></param>
        /// <returns>The hash of the new backup</returns>
        public byte[] AddBackup(BackupSetReference bsname, string message, byte[] metadatatreehash, DateTime backupTime, BackupSet?bset = null)
        {
            if (bset == null)
            {
                bset = LoadBackupSet(bsname);
            }
            BackupRecord newbackup = new(message, metadatatreehash, backupTime);

            byte[] brbytes    = newbackup.serialize();
            byte[] backuphash = BlobStore.StoreData(new List <BlobStore>(1)
            {
                Dependencies.Blobs
            }, bsname, brbytes);
            bset.Backups.Add((backuphash, bsname.Shallow));
            return(backuphash);
        }
Beispiel #10
0
        private void IncrementReferenceCountNoRecurse(BackupSetReference backupset, BlobLocation blocation, byte[] blobhash, int amount)
        {
            bool originallyshallow = blocation.TotalNonShallowReferenceCount == 0;
            int? refCount          = blocation.GetBSetReferenceCount(backupset);
            int  newRefCount       = refCount.GetValueOrDefault(0) + amount;

            blocation.SetBSetReferenceCount(backupset, newRefCount);
            if (newRefCount == 0)
            {
                blocation.RemoveBSetReference(backupset);
            }
            else if (newRefCount < 0)
            {
                throw new Exception("Negative reference count in blobstore");
            }
            if (blocation.BlockHashes == null) // Can't delete from disk if this is a multiblock reference (does not directly store data on disk)
            {
                if (blocation.TotalNonShallowReferenceCount == 0)
                {
                    if (!originallyshallow)
                    {
                        try
                        {
                            if (blocation.EncryptedHash == null)
                            {
                                throw new Exception("Hash should not be null");
                            }
                            Dependencies.DeleteBlob(blocation.EncryptedHash, blocation.RelativeFilePath);
                        }
                        catch (Exception e)
                        {
                            throw new Exception("Error deleting unreferenced file.", e);
                        }
                    }
                }
            }
            if (blocation.TotalReferenceCount == 0)
            {
                IndexStore.Remove(blobhash);
            }
        }
Beispiel #11
0
        public void RemoveBackup(BackupSetReference bsname, string backuphashprefix, bool dst_wo_cache, bool force_delete = false)
        {
            var bset = LoadBackupSet(bsname);

            if (bset.CacheUsed && dst_wo_cache && !force_delete)
            {
                // TODO: Do we need this check in more places?
                throw new Core.BackupRemoveException("Deleting a backup from a backup destination that uses a cache, " +
                                                     "without that cache present may cause errors when merging cache.");
            }
            var match = HashByPrefix(bsname, backuphashprefix);

            // TODO: Better error messages depending on return value of HashByPrefix()
            // TODO: Cleanup usage of strings vs byte[] for hashes between backup store and Core
            if (match == null || match.Value.multiplematches == true)
            {
                throw new KeyNotFoundException();
            }
            byte[] backuphash;
            if (match.Value.singlematchhash != null)
            {
                backuphash = match.Value.singlematchhash;
            }
            else
            {
                throw new Exception("HashByPrefix returned an incorrect type");
            }
            int i;

            for (i = 0; i < bset.Backups.Count; i++)
            {
                if (bset.Backups[i].hash.SequenceEqual(backuphash))
                {
                    break;
                }
            }
            Dependencies.Blobs.DecrementReferenceCount(bsname, backuphash, BlobLocation.BlobType.BackupRecord, !bset.Backups[i].shallow);
            bset.Backups.RemoveAt(i);
            SaveBackupSet(bset, bsname);
        }
Beispiel #12
0
        public void FinalizeBlobAddition(BackupSetReference bsname, byte[] blobhash, BlobLocation.BlobType blobType)
        {
            // Handle root blob
            BlobLocation rootblocation = GetBlobLocation(blobhash);

            if (rootblocation.TotalReferenceCount == 0)
            {
                IBlobReferenceIterator blobReferences = GetAllBlobReferences(blobhash, blobType, true, false);
                // Loop through children
                foreach (byte[] reference in blobReferences)
                {
                    BlobLocation blocation = GetBlobLocation(reference);
                    if (blocation.TotalReferenceCount > 0) // This was already stored
                    {
                        blobReferences.SkipChildrenOfCurrent();
                    }
                    IncrementReferenceCountNoRecurse(bsname, blocation, blobhash, 1);
                }
            }
            // Increment root blob
            IncrementReferenceCountNoRecurse(bsname, rootblocation, blobhash, 1);
        }
Beispiel #13
0
        /// <summary>
        /// Backup data sychronously.
        /// </summary>
        /// <param name="relpath"></param>
        /// <returns>A list of hashes representing the file contents.</returns>
        public static byte[] StoreData(IEnumerable <BlobStore> blobStores, BackupSetReference backupset, Stream readerbuffer)
        {
            BlockingCollection <HashBlobPair> fileblobqueue = new();

            byte[] filehash = new byte[20]; // Overall hash of file
            SplitData(readerbuffer, filehash, fileblobqueue);

            List <byte[]> blobshashes = new();

            while (!fileblobqueue.IsCompleted)
            {
                if (fileblobqueue.TryTake(out HashBlobPair? blob))
                {
                    blobStores.AsParallel().ForAll(bs => bs.AddBlob(backupset, blob));
                    blobshashes.Add(blob.Hash);
                }
            }
            if (blobshashes.Count > 1)
            {
                // Multiple blobs so create hashlist reference to reference them all together
                blobStores.AsParallel().ForAll(bs => bs.AddMultiBlobReferenceBlob(backupset, filehash, blobshashes));
            }
            return(filehash);
        }
Beispiel #14
0
 public void RemoveBSetReference(BackupSetReference backupSet)
 {
     BSetReferenceCounts.Remove(new BackupSetKey(backupSet.BackupSetName, backupSet.Shallow, backupSet.BlobListCache));
 }
Beispiel #15
0
 public void SetBSetReferenceCount(BackupSetReference backupSet, int count)
 {
     BSetReferenceCounts[new BackupSetKey(backupSet.BackupSetName, backupSet.Shallow, backupSet.BlobListCache)] = count;
 }
Beispiel #16
0
 // TODO: Update transfer logic with new reference counting logic
 public void TransferBackup(BlobStore dst, BackupSetReference dstbackupset, byte[] bblobhash, bool includefiles)
 {
     TransferBlobAndReferences(dst, dstbackupset, bblobhash, BlobLocation.BlobType.BackupRecord, includefiles);
 }
Beispiel #17
0
        // TODO: If include files is false, should we require dstbackupset.EndsWith(Core.ShallowSuffix)?
        public void TransferBlobAndReferences(BlobStore dst, BackupSetReference dstbackupset, byte[] blobhash,
                                              BlobLocation.BlobType blobtype, bool includefiles)
        {
            bool         refInDst;
            bool         shallowInDst;
            BlobLocation?rootDstBlobLocation = null;

            try
            {
                rootDstBlobLocation = dst.GetBlobLocation(blobhash);
                refInDst            = true;
                shallowInDst        = rootDstBlobLocation.TotalNonShallowReferenceCount == 0;
            }
            catch (KeyNotFoundException)
            {
                refInDst     = false;
                shallowInDst = false; // Meaningless when ref not in dst
            }

            if (!refInDst || (shallowInDst && includefiles))
            {
                byte[]? blob;
                if (refInDst)
                {
                    blob = RetrieveData(blobhash);
                }
                else
                {
                    (rootDstBlobLocation, blob) = TransferBlobNoReferences(dst, dstbackupset, blobhash, GetBlobLocation(blobhash));
                }

                IBlobReferenceIterator blobReferences = GetAllBlobReferences(blobhash, blobtype, includefiles, false);
                blobReferences.SupplyData(blob);
                foreach (var reference in blobReferences)
                {
                    bool         iterRefInDst;
                    bool         iterShallowInDst;
                    BlobLocation?dstBlobLocation = null;
                    try
                    {
                        dstBlobLocation  = dst.GetBlobLocation(reference);
                        iterRefInDst     = true;
                        iterShallowInDst = dstBlobLocation.TotalNonShallowReferenceCount == 0;
                    }
                    catch (KeyNotFoundException)
                    {
                        iterRefInDst     = false;
                        iterShallowInDst = false; // Meaningless when ref not in dst
                    }

                    if (!iterRefInDst || (iterShallowInDst && includefiles))
                    {
                        if (iterRefInDst)
                        {
                            blob = RetrieveData(reference);
                        }
                        else
                        {
                            (dstBlobLocation, blob) = TransferBlobNoReferences(dst, dstbackupset, reference, GetBlobLocation(reference));
                        }
                        blobReferences.SupplyData(blob);
                    }
                    else
                    {
                        // Dont need to increment child references if this already exists
                        blobReferences.SkipChildrenOfCurrent();
                    }

                    //if (!iterRefInDst) // Don't increment child reference if already present?
                    //{
                    // When we finish iterating over the children, increment this blob
#pragma warning disable CS8604 // Possible null reference argument.
                    blobReferences.PostOrderAction(() => dst.IncrementReferenceCountNoRecurse(dstbackupset, dstBlobLocation, reference, 1));
#pragma warning restore CS8604 // Possible null reference argument.
                    //}
                }
            }

#pragma warning disable CS8604 // Possible null reference argument.
            dst.IncrementReferenceCountNoRecurse(dstbackupset, rootDstBlobLocation, blobhash, 1);
#pragma warning restore CS8604 // Possible null reference argument.
        }
Beispiel #18
0
        private BlobLocation AddBlob(BackupSetReference backupset, HashBlobPair blob, List <byte[]>?blockreferences, bool shallow = false)
        {
            // We navigate down

            // Where we will put the blob data if we dont already have it stored
            BlobLocation posblocation;

            if (shallow)
            {
                posblocation = new BlobLocation(blockreferences);
            }
            else
            {
                if (blockreferences == null)
                {
                    if (blob.Block == null)
                    {
                        throw new Exception("Block can only be null in multirefernce blob");
                    }
                    posblocation = new BlobLocation(null, "", blob.Block.Length);
                }
                else
                {
                    posblocation = new BlobLocation(blockreferences);
                }
            }

            // Where the data is already stored if it exists
            (BlobLocation bloc, bool datastored)? existingblocstored;
            lock (this)
            {
                // Have we already stored this?
                existingblocstored = AddHash(blob.Hash, posblocation);
            }
            if (existingblocstored == null) // ExistBloc == null means posbloc was just added
            {
                if (!shallow)
                {
                    if (blockreferences == null)
                    {
                        if (blob.Block == null)
                        {
                            throw new Exception("Block can only be null in multirefernce blob");
                        }
                        (posblocation.EncryptedHash, posblocation.RelativeFilePath) = WriteBlob(blob.Hash, blob.Block);
                    }
                }
                else
                {
                    posblocation.RelativeFilePath = "";
                    posblocation.EncryptedHash    = blob.Hash;
                }
                // Dont change reference counts until finalization
                // IncrementReferenceCountNoRecurse(backupset, posblocation, blob.Hash, 1);
                return(posblocation);
            }
            else // Existbloc already stored at dst
            {
                (BlobLocation existingbloc, bool datastored) = existingblocstored.Value;
                // Is the data not already stored in the blobstore (are all references shallow thus far)?
                if (existingbloc.BlockHashes == null)
                {
                    if (!datastored)
                    {
                        // Data is not already stored
                        // Dont save if we are writing a bloblistcache
                        if (!backupset.BlobListCache)
                        {
                            // If we are saving to a cache and the bloblist cache indicates the destination has the data
                            // Then dont store, Else save
                            //BackupSetReference blobListCacheReference = backupset with { BlobListCache = true };
                            if (!(backupset.Cache &&
                                  existingbloc.GetBSetReferenceCount(backupset).HasValue))
                            {
                                if (blob.Block == null)
                                {
                                    throw new Exception("Block can only be null in multirefernce blob");
                                }
                                (existingbloc.EncryptedHash, existingbloc.RelativeFilePath) = WriteBlob(blob.Hash, blob.Block);
                            }
                        }
                    }
                }
                // Dont change reference counts until finalization
                // IncrementReferenceCountNoRecurse(backupset, existingbloc, blob.Hash, 1);
                return(existingbloc);
            }
        }
Beispiel #19
0
 /// <summary>
 ///
 /// </summary>
 /// <param name="dst"></param>
 /// <param name="blobhash"></param>
 /// <returns>True Blob exists in destination</returns>
 private (BlobLocation bloc, byte[]? blob) TransferBlobNoReferences(BlobStore dst, BackupSetReference dstbackupset,
                                                                    byte[] blobhash, BlobLocation blocation)
 {
     if (blocation.BlockHashes == null)
     {
         byte[] blob = LoadBlob(blocation, blobhash);
         return(dst.AddBlob(dstbackupset, new HashBlobPair(blobhash, blob)), blob);
     }
     else
     {
         return(dst.AddMultiBlobReferenceBlob(dstbackupset, blobhash, blocation.BlockHashes), null);
     }
 }
Beispiel #20
0
        private BlobLocation AddMultiBlobReferenceBlob(BackupSetReference backupset, byte[] hash, List <byte[]> hashlist)
        {
            HashBlobPair referenceblob = new(hash, null);

            return(AddBlob(backupset, referenceblob, hashlist));
        }
Beispiel #21
0
 private void IncrementReferenceCountNoRecurse(BackupSetReference backupset, byte[] blobhash, int amount) =>
 IncrementReferenceCountNoRecurse(backupset, GetBlobLocation(blobhash), blobhash, amount);
Beispiel #22
0
 public static byte[] StoreData(IEnumerable <BlobStore> blobStores, BackupSetReference backupset, byte[] inputdata)
 {
     return(StoreData(blobStores, backupset, new MemoryStream(inputdata)));
 }
Beispiel #23
0
 public byte[] LoadBackupSetData(BackupSetReference backupsetname)
 {
     return(DstFSInterop.LoadIndexFileAsync(backupsetname.StringRepr(), IndexFileType.BackupSet).Result);
 }
Beispiel #24
0
 public void StoreBackupSetData(BackupSetReference backupsetname, byte[] bsdata)
 {
     DstFSInterop.StoreIndexFileAsync(backupsetname.StringRepr(), IndexFileType.BackupSet, bsdata).Wait();
 }
Beispiel #25
0
 /// <summary>
 /// Add a single blob to blobstore.
 /// </summary>
 /// <param name="blob"></param>
 /// <param name="type"></param>
 /// <returns>The BlobLocation the blob is saved to.</returns>
 private BlobLocation AddBlob(BackupSetReference backupset, HashBlobPair blob)
 {
     return(AddBlob(backupset, blob, null));
 }