コード例 #1
0
 public BackupedFile(Backup backup, BackupPlanFile file, Synchronization sync)
     : this()
 {
     _Backup          = backup;
     _File            = file;
     _Synchronization = sync;
 }
コード例 #2
0
 public BackupedFile(Backup backup, BackupPlanFile file)
     : this()
 {
     _Backup = backup;
     if (_Backup != null)
     {
         if (_Backup.BackupPlan != null && _Backup.BackupPlan.StorageAccount != null)
         {
             _StorageAccountType = _Backup.BackupPlan.StorageAccountType;
             _StorageAccount     = _Backup.BackupPlan.StorageAccount;
         }
     }
     _File = file;
 }
コード例 #3
0
        private bool IsFileModified(Models.BackupPlanFile file, Models.BackupedFile lastVersion)
        {
            bool didChange = !AreEqual(file.LastWrittenAt, lastVersion.FileLastWrittenAt);

            if (!didChange)
            {
                return(false);                // If the last write dates are equal, we ASSUME it's not modified.
            }
            // If one of the checksums doesn't exist, assume the file changed.
            //if (file.LastChecksum == null || lastVersion.FileLastChecksum == null)
            //	return true;

            didChange = !AreEqual(file.LastChecksum, lastVersion.FileLastChecksum);
            return(didChange);
        }
コード例 #4
0
 public BackupPlanPathNode(BackupPlanFile planFile, EntryType type, string name, string path, BackupPlanPathNode parent)
     : this()
 {
     _StorageAccountType = planFile.StorageAccountType;
     _StorageAccount     = planFile.StorageAccount;
     _Type = type;
     // Only assign `PlanFile` if this is for a node that represents a FILE.
     if (_Type == EntryType.FILE)
     {
         _PlanFile = planFile;
     }
     _Name   = name;
     _Path   = path;
     _Parent = parent;
 }
コード例 #5
0
        //
        // Loads or creates `BackupPlanFile`s for each file in `filePaths`.
        // Returns the complete list of `BackupPlanFile`s that are related to `filePaths`.
        // If a `BackupPlanFile` does not exist for a given filePath, one will be created.
        //
        // NOTE: This method does NOT change the database.
        //
        private LinkedList <Models.BackupPlanFile> DoLoadOrCreateBackupPlanFiles(Models.BackupPlan plan, LinkedList <string> filePaths)
        {
            Assert.IsNotNull(plan);
            Assert.IsNotNull(filePaths);
            Assert.IsNotNull(AllFilesFromPlan);

            BlockPerfStats stats = new BlockPerfStats();

            stats.Begin();

            Dictionary <string, Models.BackupPlanFile> processed = new Dictionary <string, Models.BackupPlanFile>();

            // Check all files.
            foreach (string path in filePaths)
            {
                // Throw if the operation was canceled.
                CancellationToken.ThrowIfCancellationRequested();

                string normalizedPath = StringUtils.NormalizeUsingPreferredForm(path);

                //
                // Create or update `BackupPlanFile`.
                //
                Models.BackupPlanFile backupPlanFile = null;
                // The complexity of Dictionary<TKey,TValue>.TryGetValue(TKey,TValue) approaches O(1)
                bool backupPlanFileAlreadyExists = AllFilesFromPlan.TryGetValue(normalizedPath, out backupPlanFile);

                if (!backupPlanFileAlreadyExists)
                {
                    backupPlanFile           = new Models.BackupPlanFile(plan, normalizedPath);
                    backupPlanFile.CreatedAt = DateTime.UtcNow;
                }

                // This avoids duplicates in the list.
                // The complexity of setting Dictionary<TKey,TValue>[TKey] is amortized O(1)
                processed[normalizedPath] = backupPlanFile;
            }

            LinkedList <Models.BackupPlanFile> result =
                processed.ToLinkedList <Models.BackupPlanFile, KeyValuePair <string, Models.BackupPlanFile> >(p => p.Value);

            stats.End();

            return(result);
        }
コード例 #6
0
        public virtual bool Equals(BackupPlanFile other)
        {
            // If parameter is null, return false.
            if (other == null)
            {
                return(false);
            }

            bool otherIsTransient = !other.Id.HasValue;
            bool thisIsTransient  = !Id.HasValue;

            if (otherIsTransient && thisIsTransient)
            {
                return(ReferenceEquals(other, this));
            }

            return(other.Id.Equals(Id));
        }
コード例 #7
0
        //
        // REFERENCE: http://nhibernate.info/doc/patternsandpractices/identity-field-equality-and-hash-code.html
        //

        public override bool Equals(object obj)
        {
            BackupPlanFile other = obj as BackupPlanFile;

            return(this.Equals(other));
        }
コード例 #8
0
        //
        // Summary:
        // Update the `LastWrittenAt`,`LastSize`,`LastStatus`,`LastUpdatedAt`,`LastChecksum`
        // properties of each file in `files` according to the actual state of the file in the filesystem.
        //
        // NOTE: This function has a side effect - It updates properties of items from `files`.
        //
        private void DoUpdateBackupPlanFilesStatus(LinkedList <Models.BackupPlanFile> files, bool isNewVersion)
        {
            Assert.IsNotNull(files);

            ISession session = NHibernateHelper.GetSession();
            BackupedFileRepository daoBackupedFile = new BackupedFileRepository(session);

            BlockPerfStats stats = new BlockPerfStats();

            stats.Begin();

            // Check all files.
            LinkedListNode <Models.BackupPlanFile> node = files.First;

            while (node != null)
            {
                var next = node.Next;
                Models.BackupPlanFile entry = node.Value;
                // TODO(jweyrich): Measure whether `daoBackupedFile.GetLatestVersion(entry)` is faster or not,
                //                 and whether "entry.Versions.anything" would cause all related version to be fetched.
#if false
                Models.BackupedFile lastVersion = entry.Versions != null && entry.Versions.Count > 0
                                        ? entry.Versions.Last() : null;
#else
                // This may be a version that has not COMPLETED the transfer.
                Models.BackupedFile lastVersion = entry.Id.HasValue ? daoBackupedFile.GetLatestVersion(entry) : null;
#endif

                // Throw if the operation was canceled.
                CancellationToken.ThrowIfCancellationRequested();

                //
                // Check what happened to the file.
                //

                bool fileExistsOnFilesystem            = FileManager.FileExists(entry.Path);
                Models.BackupFileStatus?changeStatusTo = null;

                try
                {
                    //
                    // Update file properties
                    //
                    if (fileExistsOnFilesystem)
                    {
                        try
                        {
                            DateTime fileLastWrittenAt = FileManager.UnsafeGetFileLastWriteTimeUtc(entry.Path);
                            long     fileLength        = FileManager.UnsafeGetFileSize(entry.Path);

                            entry.LastWrittenAt = fileLastWrittenAt;
                            entry.LastSize      = fileLength;
                        }
                        catch (Exception ex)
                        {
                            string message = string.Format("Caught an exception while retrieving file properties: {0}", ex.Message);

                            Results.OnFileFailed(this, new FileVersionerEventArgs {
                                FilePath = entry.Path, FileSize = 0
                            }, message);
                            logger.Warn(message);

                            throw;
                        }

                        try
                        {
                            // Skip files larger than `MAX_FILESIZE_TO_HASH`.
                            int result = BigInteger.Compare(entry.LastSize, MAX_FILESIZE_TO_HASH);
                            if (result < 0)
                            {
                                entry.LastChecksum = CalculateHashForFile(entry.Path);
                            }
                        }
                        catch (Exception ex)
                        {
                            string message = string.Format("Caught an exception while calculating the file hash: {0}", ex.Message);

                            Results.OnFileFailed(this, new FileVersionerEventArgs {
                                FilePath = entry.Path, FileSize = 0
                            }, message);
                            logger.Warn(message);

                            throw;
                        }

                        Results.OnFileCompleted(this, new FileVersionerEventArgs {
                            FilePath = entry.Path, FileSize = entry.LastSize
                        });
                    }

                    //
                    // Update file status
                    //
                    if (lastVersion != null)                     // File was backed up at least once in the past?
                    {
                        switch (entry.LastStatus)
                        {
                        case Models.BackupFileStatus.DELETED:                               // File was marked as DELETED by a previous backup?
                            if (fileExistsOnFilesystem)                                     // Exists?
                            {
                                changeStatusTo = Models.BackupFileStatus.ADDED;
                            }
                            break;

                        case Models.BackupFileStatus.REMOVED:                               // File was marked as REMOVED by a previous backup?
                            if (fileExistsOnFilesystem)                                     // Exists?
                            {
                                changeStatusTo = Models.BackupFileStatus.ADDED;
                            }
                            else
                            {
                                // QUESTION: Do we really care to transition REMOVED to DELETED?
                                changeStatusTo = Models.BackupFileStatus.DELETED;
                            }
                            break;

                        default:                                 // ADDED, MODIFIED, UNCHANGED
                            if (fileExistsOnFilesystem)          // Exists?
                            {
                                // DO NOT verify whether the file changed for a `ResumeBackupOperation`,
                                // only for `NewBackupOperation`.
                                if (isNewVersion)
                                {
                                    if (IsFileModified(entry, lastVersion))                                             // Modified?
                                    {
                                        changeStatusTo = Models.BackupFileStatus.MODIFIED;
                                    }
                                    else if (NeedsToRetryFile(entry, lastVersion))                                             // Didn't complete last file transfer?
                                    {
                                        changeStatusTo = Models.BackupFileStatus.MODIFIED;
                                    }
                                    else                                             // Not modified?
                                    {
                                        changeStatusTo = Models.BackupFileStatus.UNCHANGED;
                                    }
                                }
                            }
                            else                                     // Deleted from filesystem?
                            {
                                changeStatusTo = Models.BackupFileStatus.DELETED;
                            }
                            break;
                        }
                    }
                    else                            // Adding to this backup?
                    {
                        if (fileExistsOnFilesystem) // Exists?
                        {
                            changeStatusTo = Models.BackupFileStatus.ADDED;
                        }
                        else
                        {
                            // Error? Can't add a non-existent file to the plan.
                        }
                    }

                    if (changeStatusTo.HasValue)
                    {
                        entry.LastStatus = changeStatusTo.Value;
                        entry.UpdatedAt  = DateTime.UtcNow;
                    }
                }
                catch (Exception ex)
                {
                    FailedFile <Models.BackupPlanFile> failedEntry = new FailedFile <Models.BackupPlanFile>(entry, ex.Message, ex);
                    ChangeSet.FailedFiles.AddLast(failedEntry);

                    // Remove this entry from `files` as it clearly failed.
                    files.Remove(node);                     // Complexity is O(1)
                }

                node = next;
            }

            stats.End();
        }
コード例 #9
0
 //
 // Summary:
 // Returns true if:
 // - File transfer didn't begin;
 // - File transfer didn't complete;
 // - File transfer failed;
 // - File transfer was canceled;
 //
 private bool NeedsToRetryFile(Models.BackupPlanFile file, Models.BackupedFile lastVersion)
 {
     return(lastVersion.TransferStatus != TransferStatus.COMPLETED &&
            lastVersion.TransferStatus != TransferStatus.PURGED);
 }
コード例 #10
0
        // Summary:
        //    Saves all instances from RemoteObjects list to the database.
        //    Also removes them from RemoteObjects list to free memory.
        private void Save(CancellationToken CancellationToken)
        {
            ISession session = NHibernateHelper.GetSession();

            BatchProcessor               batchProcessor        = new BatchProcessor(250);
            StorageAccountRepository     daoStorageAccount     = new StorageAccountRepository(session);
            BackupPlanFileRepository     daoBackupPlanFile     = new BackupPlanFileRepository(session);
            BackupPlanPathNodeRepository daoBackupPlanPathNode = new BackupPlanPathNodeRepository(session);
            BackupedFileRepository       daoBackupedFile       = new BackupedFileRepository(session);

            BlockPerfStats stats = new BlockPerfStats();

            using (BatchTransaction tx = batchProcessor.BeginTransaction(session))
            {
                try
                {
                    // ------------------------------------------------------------------------------------

                    Models.StorageAccount account = daoStorageAccount.Get(Synchronization.StorageAccount.Id);

                    // ------------------------------------------------------------------------------------

                    stats.Begin("STEP 1");

                    BackupPlanPathNodeCreator pathNodeCreator = new BackupPlanPathNodeCreator(daoBackupPlanPathNode, tx);

                    // Report save progress
                    ReportSaveProgress(SyncAgent.Results.Stats.SavedFileCount, true);

                    // Saving loop
                    for (int i = RemoteObjects.Count - 1; i >= 0; i--)
                    {
                        ListingObject obj = RemoteObjects[i];              // Get instance of object.
                        //RemoteObjects[i] = null;
                        RemoteObjects.RemoveAt(i);                         // Remove to free memory. RemoveAt(int) is O(N).

                        // Throw if the operation was canceled.
                        CancellationToken.ThrowIfCancellationRequested();

                        Models.EntryType type;
                        string           path          = string.Empty;
                        string           versionString = string.Empty;

                        try
                        {
                            // Parse obj.Key into its relevant parts.
                            bool ok = ParseS3Key(obj.Key, out type, out path, out versionString);
                        }
                        catch (Exception ex)
                        {
                            if (ex is ArgumentException || ex is IndexOutOfRangeException)
                            {
                                // Report error.
                                logger.Warn("Failed to parse S3 key: {0} -- Skipping.", obj.Key);
                                //logger.Log(LogLevel.Warn, ex, "Failed to parse S3 key: {0}", obj.Key);

                                //SyncAgent.Results.Stats.FailedSavedFileCount += 1;

                                // Report save progress
                                //ReportSaveProgress(SyncAgent.Results.Stats.SavedFileCount);

                                continue;                                 // Skip this file.
                            }

                            throw;
                        }

                        path = StringUtils.NormalizeUsingPreferredForm(path);

                        DateTime lastWrittenAt = DateTime.ParseExact(versionString, Models.BackupedFile.VersionFormat, CultureInfo.InvariantCulture);

                        // Create/Update BackupPlanFile, but do not SAVE it.
                        Models.BackupPlanFile entry   = daoBackupPlanFile.GetByStorageAccountAndPath(account, path);
                        Models.BackupedFile   version = null;

                        if (entry == null)
                        {
                            // Create `BackupPlanFile`.
                            entry                    = new Models.BackupPlanFile();
                            entry.BackupPlan         = null;
                            entry.StorageAccountType = account.Type;
                            entry.StorageAccount     = account;
                            entry.Path               = path;
                            entry.LastSize           = obj.Size;
                            entry.LastWrittenAt      = lastWrittenAt;
                            //entry.LastChecksum = ;
                            entry.LastStatus = Models.BackupFileStatus.UNCHANGED;
                            entry.CreatedAt  = DateTime.UtcNow;

                            // Create `BackupedFile`.
                            version = new Models.BackupedFile(null, entry, Synchronization);
                            version.StorageAccountType = account.Type;
                            version.StorageAccount     = account;
                            version.FileLastWrittenAt  = lastWrittenAt;
                            version.FileLastChecksum   = entry.LastChecksum;
                            version.FileSize           = entry.LastSize;
                            version.FileStatus         = Models.BackupFileStatus.MODIFIED;
                            version.TransferStatus     = TransferStatus.COMPLETED;
                            version.UpdatedAt          = DateTime.UtcNow;

                            entry.Versions.Add(version);
                            //daoBackupedFile.Insert(tx, version);
                        }
                        else
                        {
                            // Update `BackupPlanFile`.
                            entry.LastSize      = obj.Size;
                            entry.LastWrittenAt = lastWrittenAt;
                            //entry.LastChecksum =
                            //entry.LastStatus = Models.BackupFileStatus.MODIFIED;
                            entry.UpdatedAt = DateTime.UtcNow;

                            IList <Models.BackupedFile> versions = null;
                            try
                            {
                                versions = daoBackupedFile.GetCompletedByStorageAccountAndPath(account, path, versionString);
                            }
                            catch (FormatException)
                            {
                                // Report error.
                                logger.Warn("Failed to parse versionString: {0} -- Skipping.", versionString);

                                //SyncAgent.Results.Stats.FailedSavedFileCount += 1;

                                continue;                                 // TODO(jweyrich): Should we abort?
                            }

                            // Check whether our database already contains this exact file + version.
                            if (versions == null || (versions != null && versions.Count == 0))
                            {
                                // Create `BackupedFile`.
                                version = new Models.BackupedFile(null, entry, Synchronization);
                                version.StorageAccountType = account.Type;
                                version.StorageAccount     = account;
                                version.FileLastWrittenAt  = entry.LastWrittenAt;
                                version.FileLastChecksum   = entry.LastChecksum;
                                version.FileSize           = entry.LastSize;
                                version.FileStatus         = Models.BackupFileStatus.MODIFIED;
                                version.TransferStatus     = TransferStatus.COMPLETED;
                                version.UpdatedAt          = DateTime.UtcNow;

                                entry.Versions.Add(version);
                                //daoBackupedFile.Insert(tx, version);
                            }
                            else
                            {
                                // Update `BackupedFile`.
                                version = versions.First();
                                version.FileLastWrittenAt = entry.LastWrittenAt;
                                version.FileLastChecksum  = entry.LastChecksum;
                                version.FileSize          = entry.LastSize;
                                version.UpdatedAt         = DateTime.UtcNow;
                                //daoBackupedFile.Update(tx, version);
                            }
                        }

                        try
                        {
                            // Create path nodes and INSERT them, if they don't exist yet.
                            entry.PathNode = pathNodeCreator.CreateOrUpdatePathNodes(account, entry);

                            // Create or update `BackupPlanFile`.
                            daoBackupPlanFile.InsertOrUpdate(tx, entry);
                        }
                        catch (Exception ex)
                        {
                            logger.Log(LogLevel.Error, ex, "BUG: Failed to insert/update {0} => {1}",
                                       typeof(Models.BackupPlanFile).Name,
                                       CustomJsonSerializer.SerializeObject(entry, 1));

                            logger.Error("Dump of failed object: {0}", entry.DumpMe());
                            throw;
                        }

                        bool didCommit = batchProcessor.ProcessBatch(tx);

                        SyncAgent.Results.Stats.SavedFileCount += 1;

                        // Report save progress
                        ReportSaveProgress(SyncAgent.Results.Stats.SavedFileCount);
                    }

                    batchProcessor.ProcessBatch(tx, true);

                    // Report save progress
                    ReportSaveProgress(SyncAgent.Results.Stats.SavedFileCount, true);

                    stats.End();

                    // ------------------------------------------------------------------------------------

                    tx.Commit();
                }
                catch (OperationCanceledException)
                {
                    tx.Rollback();                     // Rollback the transaction
                    throw;
                }
                catch (Exception ex)
                {
                    logger.Log(LogLevel.Error, ex, "Caught exception");
                    tx.Rollback();                     // Rollback the transaction
                    throw;
                }
                finally
                {
                    //session.Close();
                    if (session.IsConnected)
                    {
                        session.Disconnect();
                    }
                }
            }
        }
コード例 #11
0
        public Models.BackupPlanPathNode CreateOrUpdatePathNodes(Models.StorageAccount account, Models.BackupPlanFile file)
        {
            PathNodes pathNodes = new PathNodes(file.Path);

            bool nodeExists = true;             // Start assuming it exists.

            Models.BackupPlanPathNode previousNode = null;
            Models.BackupPlanPathNode planPathNode = null;
            foreach (var pathNode in pathNodes.Nodes)
            {
                // If it does not exist, it does not make sense to lookup inner directories/files.
                if (nodeExists)
                {
                    planPathNode = _dao.GetByStorageAccountAndTypeAndPath(
                        account, Models.EntryTypeExtensions.ToEntryType(pathNode.Type), pathNode.Path);

                    // If we couldn't find the current `Models.BackupPlanPathNode`, it's safe to assume the inner
                    // directories/files don't exist either. From now on, all nodes will be created/inserted.
                    if (planPathNode == null)
                    {
                        nodeExists = false;
                    }
                }

                if (!nodeExists)
                {
                    //BackupPlanFile planFile = daoBackupPlanFile.GetByPlanAndPath(Backup.BackupPlan, file.Path);
                    //Assert.NotNull(planFile, string.Format("Required {0} not found in the database.", typeof(BackupPlanFile).Name))
                    planPathNode = new Models.BackupPlanPathNode(file,
                                                                 Models.EntryTypeExtensions.ToEntryType(pathNode.Type),
                                                                 pathNode.Name, pathNode.Path, previousNode);

                    if (previousNode != null)
                    {
                        planPathNode.Parent = previousNode;
                        previousNode.SubNodes.Add(planPathNode);
                    }


                    _dao.Insert(_tx, planPathNode);
                    _dao.Refresh(planPathNode);
                }

                previousNode = planPathNode;

                //session.Evict(planPathNode); // Force future queries to re-load it and its relationships.
            }

            return(previousNode);
        }