示例#1
0
        private void UploadToGoogleCloud(GoogleCloudSettings settings, Stream stream, Progress progress)
        {
            using (var client = new RavenGoogleCloudClient(settings, _settings.Configuration, progress, TaskCancelToken.Token))
            {
                var key = CombinePathAndKey(settings.RemoteFolderName);
                client.UploadObject(key, stream, new Dictionary <string, string>
                {
                    { "Description", GetArchiveDescription() }
                });

                if (_logger.IsInfoEnabled)
                {
                    _logger.Info($"{ReportSuccess(GoogleCloudName)} storage bucket: {settings.BucketName}");
                }

                if (_retentionPolicyParameters == null)
                {
                    return;
                }

                var runner = new GoogleCloudRetentionPolicyRunner(_retentionPolicyParameters, client);
                runner.Execute();
            }
        }
示例#2
0
        private void CreateUploadTaskIfNeeded <S, T>(
            S settings,
            List <Task> tasks,
            string backupPath,
            bool isFullBackup,
            Func <S, FileStream, Progress, Task> uploadToServer,
            T uploadStatus,
            Action <IOperationProgress> onProgress)
            where S : BackupSettings
            where T : CloudUploadStatus
        {
            if (PeriodicBackupConfiguration.CanBackupUsing(settings) == false)
            {
                return;
            }

            Debug.Assert(uploadStatus != null);

            var localUploadStatus = uploadStatus;

            tasks.Add(Task.Run(async() =>
            {
                using (localUploadStatus.UpdateStats(isFullBackup))
                    using (var fileStream = File.OpenRead(backupPath))
                    {
                        var uploadProgress        = localUploadStatus.UploadProgress;
                        localUploadStatus.Skipped = false;
                        uploadProgress.ChangeState(UploadState.PendingUpload);
                        uploadProgress.SetTotal(fileStream.Length);

                        AddInfo($"Starting {uploadStatus.GetType().AssemblyQualifiedName}", onProgress);

                        try
                        {
                            var bytesPutsPerSec = new MeterMetric();

                            long lastUploadedInBytes = 0;
                            var totalToUpload        = new Sparrow.Size(uploadProgress.TotalInBytes, SizeUnit.Bytes).ToString();
                            var sw       = Stopwatch.StartNew();
                            var progress = new Progress(uploadProgress)
                            {
                                OnUploadProgress = () =>
                                {
                                    if (sw.ElapsedMilliseconds <= 1000)
                                    {
                                        return;
                                    }

                                    var totalUploadedInBytes = uploadProgress.UploadedInBytes;
                                    bytesPutsPerSec.MarkSingleThreaded(totalUploadedInBytes - lastUploadedInBytes);
                                    lastUploadedInBytes            = totalUploadedInBytes;
                                    var uploaded                   = new Sparrow.Size(totalUploadedInBytes, SizeUnit.Bytes);
                                    uploadProgress.BytesPutsPerSec = bytesPutsPerSec.MeanRate;
                                    AddInfo($"Uploaded: {uploaded} / {totalToUpload}", onProgress);
                                    sw.Restart();
                                }
                            };

                            await uploadToServer(settings, fileStream, progress);

                            AddInfo($"Total uploaded: {totalToUpload}, " +
                                    $"took: {MsToHumanReadableString(uploadProgress.UploadTimeInMs)}", onProgress);
                        }
                        catch (OperationCanceledException e)
                        {
                            // shutting down
                            localUploadStatus.Exception = e.ToString();
                            throw;
                        }
                        catch (Exception e)
                        {
                            localUploadStatus.Exception = e.ToString();
                            throw new InvalidOperationException($"Failed to backup to {uploadStatus.GetType().FullName}", e);
                        }
                        finally
                        {
                            uploadProgress.ChangeState(UploadState.Done);
                        }
                    }
            }));
        }
示例#3
0
        private void CreateUploadTaskIfNeeded <S, T>(S settings, Action <S, FileStream, Progress> uploadToServer, T uploadStatus, string targetName)
            where S : BackupSettings
            where T : CloudUploadStatus
        {
            if (PeriodicBackupConfiguration.CanBackupUsing(settings) == false)
            {
                return;
            }

            Debug.Assert(uploadStatus != null);

            var localUploadStatus = uploadStatus;
            var thread            = PoolOfThreads.GlobalRavenThreadPool.LongRunning(_ =>
            {
                try
                {
                    Thread.CurrentThread.Priority = ThreadPriority.BelowNormal;
                    NativeMemory.EnsureRegistered();

                    using (localUploadStatus.UpdateStats(_isFullBackup))
                        using (var fileStream = File.OpenRead(_backupUploaderSettings.BackupPath))
                        {
                            var uploadProgress = localUploadStatus.UploadProgress;
                            try
                            {
                                localUploadStatus.Skipped = false;
                                uploadProgress.ChangeState(UploadState.PendingUpload);
                                uploadProgress.SetTotal(fileStream.Length);

                                AddInfo($"Starting the upload of backup file to {targetName}.");

                                var bytesPutsPerSec = new MeterMetric();

                                long lastUploadedInBytes = 0;
                                var totalToUpload        = new Size(uploadProgress.TotalInBytes, SizeUnit.Bytes).ToString();
                                var sw       = Stopwatch.StartNew();
                                var progress = new Progress(uploadProgress)
                                {
                                    OnUploadProgress = () =>
                                    {
                                        if (sw.ElapsedMilliseconds <= 1000)
                                        {
                                            return;
                                        }

                                        var totalUploadedInBytes = uploadProgress.UploadedInBytes;
                                        bytesPutsPerSec.MarkSingleThreaded(totalUploadedInBytes - lastUploadedInBytes);
                                        lastUploadedInBytes            = totalUploadedInBytes;
                                        var uploaded                   = new Size(totalUploadedInBytes, SizeUnit.Bytes);
                                        uploadProgress.BytesPutsPerSec = bytesPutsPerSec.MeanRate;
                                        AddInfo($"Uploaded: {uploaded} / {totalToUpload}");
                                        sw.Restart();
                                    }
                                };

                                uploadToServer(settings, fileStream, progress);

                                AddInfo($"Total uploaded: {totalToUpload}, took: {MsToHumanReadableString(uploadProgress.UploadTimeInMs)}");
                            }
                            finally
                            {
                                uploadProgress.ChangeState(UploadState.Done);
                            }
                        }
                }
                catch (OperationCanceledException e)
                {
                    // shutting down
                    localUploadStatus.Exception = e.ToString();
                    _exceptions.Add(e);
                }
                catch (Exception e)
                {
                    localUploadStatus.Exception = e.ToString();
                    _exceptions.Add(new InvalidOperationException($"Failed to upload the backup file to {targetName}.", e));
                }
            }, null, $"Upload backup file of database '{_backupUploaderSettings.DatabaseName}' to {targetName} (task: '{_backupUploaderSettings.TaskName}')");

            _threads.Add(thread);
        }
 public ProgressableStreamContent(Stream content, Progress progress)
 {
     _content  = content ?? throw new ArgumentNullException("content");
     _progress = progress;
 }