Esempio n. 1
0
        public void Run(string[] sources, Library.Utility.IFilter filter)
        {
            m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Begin);

            using(m_database = new LocalBackupDatabase(m_options.Dbpath, m_options))
            {
                m_result.SetDatabase(m_database);
                m_result.Dryrun = m_options.Dryrun;

                Utility.UpdateOptionsFromDb(m_database, m_options);
                Utility.VerifyParameters(m_database, m_options);

                m_blocksize = m_options.Blocksize;

                m_blockbuffer = new byte[m_options.Blocksize * Math.Max(1, m_options.FileReadBufferSize / m_options.Blocksize)];
                m_blocklistbuffer = new byte[m_options.Blocksize];

                m_blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
                m_filehasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.FileHashAlgorithm);

                if (m_blockhasher == null)
                    throw new Exception(Strings.Foresthash.InvalidHashAlgorithm(m_options.BlockHashAlgorithm));
                if (m_filehasher == null)
                    throw new Exception(Strings.Foresthash.InvalidHashAlgorithm(m_options.FileHashAlgorithm));

                if (!m_blockhasher.CanReuseTransform)
                    throw new Exception(Strings.Foresthash.InvalidCryptoSystem(m_options.BlockHashAlgorithm));
                if (!m_filehasher.CanReuseTransform)
                    throw new Exception(Strings.Foresthash.InvalidCryptoSystem(m_options.FileHashAlgorithm));

                m_database.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize);
                // If there is no filter, we set an empty filter to simplify the code
                // If there is a filter, we make sure that the sources are included
                m_filter = filter ?? new Library.Utility.FilterExpression();
                m_sourceFilter = new Library.Utility.FilterExpression(sources, true);

                var lastVolumeSize = -1L;
                m_backendLogFlushTimer = DateTime.Now.Add(FLUSH_TIMESPAN);
                System.Threading.Thread parallelScanner = null;

                try
                {
                    m_snapshot = GetSnapshot(sources, m_options, m_result);

                    // Start parallel scan
                    if (m_options.ChangedFilelist == null || m_options.ChangedFilelist.Length < 1)
                    {
                        parallelScanner = new System.Threading.Thread(CountFilesThread) {
                            Name = "Read ahead file counter",
                            IsBackground = true
                        };
                        parallelScanner.Start();
                    }

                    using(m_backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, m_database))
                    using(m_filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp))
                    {
                        m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PreBackupVerify);
                        using(new Logging.Timer("PreBackupVerify"))
                        {
                            try
                            {
                                if (m_options.NoBackendverification)
                                    FilelistProcessor.VerifyLocalList(m_backend, m_options, m_database, m_result.BackendWriter);
                                else
                                    FilelistProcessor.VerifyRemoteList(m_backend, m_options, m_database, m_result.BackendWriter);
                            }
                            catch (Exception ex)
                            {
                                if (m_options.AutoCleanup)
                                {
                                    m_result.AddWarning("Backend verification failed, attempting automatic cleanup", ex);
                                    m_result.RepairResults = new RepairResults(m_result);
                                    new RepairHandler(m_backend.BackendUrl, m_options, (RepairResults)m_result.RepairResults).Run();

                                    m_result.AddMessage("Backend cleanup finished, retrying verification");
                                    FilelistProcessor.VerifyRemoteList(m_backend, m_options, m_database, m_result.BackendWriter);
                                }
                                else
                                    throw;
                            }
                        }

                        // Verify before uploading a synthetic list
                        m_database.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize);
                        UploadSyntheticFilelist();

                        m_database.BuildLookupTable(m_options);
                        m_transaction = m_database.BeginTransaction();

                        m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_ProcessingFiles);
                        var filesetvolumeid = m_database.RegisterRemoteVolume(m_filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, m_transaction);
                        m_database.CreateFileset(filesetvolumeid, VolumeBase.ParseFilename(m_filesetvolume.RemoteFilename).Time, m_transaction);

                        m_blockvolume = new BlockVolumeWriter(m_options);
                        m_blockvolume.VolumeID = m_database.RegisterRemoteVolume(m_blockvolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, m_transaction);

                        if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
                        {
                            m_indexvolume = new IndexVolumeWriter(m_options);
                            m_indexvolume.VolumeID = m_database.RegisterRemoteVolume(m_indexvolume.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, m_transaction);
                        }

                        var filterhandler = new FilterHandler(m_snapshot, m_attributeFilter, m_sourceFilter, m_filter, m_symlinkPolicy, m_options.HardlinkPolicy, m_result);

                        using(new Logging.Timer("BackupMainOperation"))
                        {
                            if (m_options.ChangedFilelist != null && m_options.ChangedFilelist.Length >= 1)
                            {
                                m_result.AddVerboseMessage("Processing supplied change list instead of enumerating filesystem");
                                m_result.OperationProgressUpdater.UpdatefileCount(m_options.ChangedFilelist.Length, 0, true);

                                foreach(var p in filterhandler.Mixin(m_options.ChangedFilelist))
                                {
                                    if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
                                    {
                                        m_result.AddMessage("Stopping backup operation on request");
                                        break;
                                    }

                                    try
                                    {
                                        this.HandleFilesystemEntry(p, m_snapshot.GetAttributes(p));
                                    }
                                    catch (Exception ex)
                                    {
                                        m_result.AddWarning(string.Format("Failed to process element: {0}, message: {1}", p, ex.Message), ex);
                                    }
                                }

                                m_database.AppendFilesFromPreviousSet(m_transaction, m_options.DeletedFilelist);
                            }
                            else
                            {
                                foreach(var path in filterhandler.EnumerateFilesAndFolders())
                                {
                                    if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
                                    {
                                        m_result.AddMessage("Stopping backup operation on request");
                                        break;
                                    }

                                    var fa = FileAttributes.Normal;
                                    try { fa = m_snapshot.GetAttributes(path); }
                                    catch { }

                                    this.HandleFilesystemEntry(path, fa);
                                }

                            }

                            //If the scanner is still running for some reason, make sure we kill it now
                            if (parallelScanner != null && parallelScanner.IsAlive)
                                parallelScanner.Abort();

                            // We no longer need to snapshot active
                            try { m_snapshot.Dispose(); }
                            finally { m_snapshot = null; }

                            m_result.OperationProgressUpdater.UpdatefileCount(m_result.ExaminedFiles, m_result.SizeOfExaminedFiles, true);
                        }

                        m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Finalize);
                        using(new Logging.Timer("FinalizeRemoteVolumes"))
                        {
                            if (m_blockvolume.SourceSize > 0)
                            {
                                lastVolumeSize = m_blockvolume.SourceSize;

                                if (m_options.Dryrun)
                                {
                                    m_result.AddDryrunMessage(string.Format("Would upload block volume: {0}, size: {1}", m_blockvolume.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(m_blockvolume.LocalFilename).Length)));
                                    if (m_indexvolume != null)
                                    {
                                        m_blockvolume.Close();
                                        UpdateIndexVolume();
                                        m_indexvolume.FinishVolume(Library.Utility.Utility.CalculateHash(m_blockvolume.LocalFilename), new FileInfo(m_blockvolume.LocalFilename).Length);
                                        m_result.AddDryrunMessage(string.Format("Would upload index volume: {0}, size: {1}", m_indexvolume.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(m_indexvolume.LocalFilename).Length)));
                                    }

                                    m_blockvolume.Dispose();
                                    m_blockvolume = null;
                                    m_indexvolume.Dispose();
                                    m_indexvolume = null;
                                }
                                else
                                {
                                    m_database.UpdateRemoteVolume(m_blockvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_transaction);
                                    m_blockvolume.Close();
                                    UpdateIndexVolume();

                                    using(new Logging.Timer("CommitUpdateRemoteVolume"))
                                        m_transaction.Commit();
                                    m_transaction = m_database.BeginTransaction();

                                    m_backend.Put(m_blockvolume, m_indexvolume);

                                    using(new Logging.Timer("CommitUpdateRemoteVolume"))
                                        m_transaction.Commit();
                                    m_transaction = m_database.BeginTransaction();

                                    m_blockvolume = null;
                                    m_indexvolume = null;
                                }
                            }
                            else
                            {
                                m_database.RemoveRemoteVolume(m_blockvolume.RemoteFilename, m_transaction);
                                if (m_indexvolume != null)
                                    m_database.RemoveRemoteVolume(m_indexvolume.RemoteFilename, m_transaction);
                            }
                        }

                        using(new Logging.Timer("UpdateChangeStatistics"))
                            m_database.UpdateChangeStatistics(m_result);
                        using(new Logging.Timer("VerifyConsistency"))
                            m_database.VerifyConsistency(m_transaction, m_options.Blocksize, m_options.BlockhashSize);

                        var changeCount =
                            m_result.AddedFiles + m_result.ModifiedFiles + m_result.DeletedFiles +
                            m_result.AddedFolders + m_result.ModifiedFolders + m_result.DeletedFolders +
                            m_result.AddedSymlinks + m_result.ModifiedSymlinks + m_result.DeletedSymlinks;

                        //Changes in the filelist triggers a filelist upload
                        if (m_options.UploadUnchangedBackups || changeCount > 0)
                        {
                            using(new Logging.Timer("Uploading a new fileset"))
                            {
                                if (!string.IsNullOrEmpty(m_options.ControlFiles))
                                    foreach(var p in m_options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
                                        m_filesetvolume.AddControlFile(p, m_options.GetCompressionHintFromFilename(p));

                                m_database.WriteFileset(m_filesetvolume, m_transaction);
                                m_filesetvolume.Close();

                                if (m_options.Dryrun)
                                    m_result.AddDryrunMessage(string.Format("Would upload fileset volume: {0}, size: {1}", m_filesetvolume.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(m_filesetvolume.LocalFilename).Length)));
                                else
                                {
                                    m_database.UpdateRemoteVolume(m_filesetvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_transaction);

                                    using(new Logging.Timer("CommitUpdateRemoteVolume"))
                                        m_transaction.Commit();
                                    m_transaction = m_database.BeginTransaction();

                                    m_backend.Put(m_filesetvolume);

                                    using(new Logging.Timer("CommitUpdateRemoteVolume"))
                                        m_transaction.Commit();
                                    m_transaction = m_database.BeginTransaction();

                                }
                            }
                        }
                        else
                        {
                            m_result.AddVerboseMessage("removing temp files, as no data needs to be uploaded");
                            m_database.RemoveRemoteVolume(m_filesetvolume.RemoteFilename, m_transaction);
                        }

                        m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload);
                        using(new Logging.Timer("Async backend wait"))
                            m_backend.WaitForComplete(m_database, m_transaction);

                        if (m_result.TaskControlRendevouz() != TaskControlState.Stop)
                        {
                            if (m_options.KeepTime.Ticks > 0 || m_options.KeepVersions != 0)
                            {
                                m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Delete);
                                m_result.DeleteResults = new DeleteResults(m_result);
                                using(var db = new LocalDeleteDatabase(m_database))
                                    new DeleteHandler(m_backend.BackendUrl, m_options, (DeleteResults)m_result.DeleteResults).DoRun(db, m_transaction, true, lastVolumeSize <= m_options.SmallFileSize);

                            }
                            else if (lastVolumeSize <= m_options.SmallFileSize && !m_options.NoAutoCompact)
                            {
                                m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Compact);
                                m_result.CompactResults = new CompactResults(m_result);
                                using(var db = new LocalDeleteDatabase(m_database))
                                    new CompactHandler(m_backend.BackendUrl, m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, m_transaction);
                            }
                        }

                        if (m_options.UploadVerificationFile)
                        {
                            m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload);
                            FilelistProcessor.UploadVerificationFile(m_backend.BackendUrl, m_options, m_result.BackendWriter, m_database, m_transaction);
                        }

                        if (m_options.Dryrun)
                        {
                            m_transaction.Rollback();
                            m_transaction = null;
                        }
                        else
                        {
                            using(new Logging.Timer("CommitFinalizingBackup"))
                                m_transaction.Commit();

                            m_transaction = null;
                            m_database.Vacuum();

                            if (m_result.TaskControlRendevouz() != TaskControlState.Stop && !m_options.NoBackendverification)
                            {
                                m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupVerify);
                                using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, m_database))
                                {
                                    using(new Logging.Timer("AfterBackupVerify"))
                                        FilelistProcessor.VerifyRemoteList(backend, m_options, m_database, m_result.BackendWriter);
                                    backend.WaitForComplete(m_database, null);
                                }

                                if (m_options.BackupTestSampleCount > 0 && m_database.GetRemoteVolumes().Count() > 0)
                                {
                                    m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupTest);
                                    m_result.TestResults = new TestResults(m_result);

                                    using(var testdb = new LocalTestDatabase(m_database))
                                    using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, testdb))
                                        new TestHandler(m_backendurl, m_options, new TestResults(m_result))
                                            .DoRun(m_options.BackupTestSampleCount, testdb, backend);
                                }
                            }

                        }

                        m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete);
                        m_database.WriteResults();
                        m_database.PurgeLogData(m_options.LogRetention);
                        return;
                    }
                }
                catch (Exception ex)
                {
                    m_result.AddError("Fatal error", ex);
                    throw;
                }
                finally
                {
                    if (parallelScanner != null && parallelScanner.IsAlive)
                    {
                        parallelScanner.Abort();
                        parallelScanner.Join(500);
                        if (parallelScanner.IsAlive)
                            m_result.AddWarning("Failed to terminate filecounter thread", null);
                    }

                    if (m_snapshot != null)
                        try { m_snapshot.Dispose(); }
                        catch (Exception ex) { m_result.AddError(string.Format("Failed to dispose snapshot"), ex); }
                        finally { m_snapshot = null; }

                    if (m_transaction != null)
                        try { m_transaction.Rollback(); }
                        catch (Exception ex) { m_result.AddError(string.Format("Rollback error: {0}", ex.Message), ex); }
                }
            }
        }
Esempio n. 2
0
        private void RunMainOperation(Snapshots.ISnapshotService snapshot, BackendManager backend)
        {
            var filterhandler = new FilterHandler(snapshot, m_attributeFilter, m_sourceFilter, m_filter, m_symlinkPolicy, m_options.HardlinkPolicy, m_result);

            using(new Logging.Timer("BackupMainOperation"))
            {
                if (m_options.ChangedFilelist != null && m_options.ChangedFilelist.Length >= 1)
                {
                    m_result.AddVerboseMessage("Processing supplied change list instead of enumerating filesystem");
                    m_result.OperationProgressUpdater.UpdatefileCount(m_options.ChangedFilelist.Length, 0, true);

                    foreach(var p in filterhandler.Mixin(m_options.ChangedFilelist))
                    {
                        if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
                        {
                            m_result.AddMessage("Stopping backup operation on request");
                            break;
                        }

                        try
                        {
                            this.HandleFilesystemEntry(snapshot, backend, p, snapshot.GetAttributes(p));
                        }
                        catch (Exception ex)
                        {
                            m_result.AddWarning(string.Format("Failed to process element: {0}, message: {1}", p, ex.Message), ex);
                        }
                    }

                    m_database.AppendFilesFromPreviousSet(m_transaction, m_options.DeletedFilelist);
                }
                else
                {                                    
                    foreach(var path in filterhandler.EnumerateFilesAndFolders())
                    {
                        if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
                        {
                            m_result.AddMessage("Stopping backup operation on request");
                            break;
                        }

                        var fa = FileAttributes.Normal;
                        try { fa = snapshot.GetAttributes(path); }
                        catch { }

                        this.HandleFilesystemEntry(snapshot, backend, path, fa);
                    }

                }

                m_result.OperationProgressUpdater.UpdatefileCount(m_result.ExaminedFiles, m_result.SizeOfExaminedFiles, true);
            }
        }