public async Task GetImport() { if (HttpContext.Request.Query.ContainsKey("file") == false && HttpContext.Request.Query.ContainsKey("url") == false) { throw new ArgumentException("'file' or 'url' are mandatory when using GET /smuggler/import"); } using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { var options = DatabaseSmugglerOptionsServerSide.Create(HttpContext); using (var stream = new GZipStream(new BufferedStream(await GetImportStream(), 128 * Voron.Global.Constants.Size.Kilobyte), CompressionMode.Decompress)) using (var token = CreateOperationToken()) using (var source = new StreamSource(stream, context, Database)) { var destination = new DatabaseDestination(Database); var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, token: token.Token); var result = smuggler.Execute(); WriteImportResult(context, result, ResponseBodyStream()); } } }
private async Task MigrateDatabase(long operationId, ImportInfo importInfo) { var startDocumentEtag = importInfo?.LastEtag ?? 0; var url = $"{ServerUrl}/databases/{DatabaseName}/smuggler/export?operationId={operationId}&startEtag={startDocumentEtag}"; var json = JsonConvert.SerializeObject(new DatabaseSmugglerOptionsServerSide()); var content = new StringContent(json, Encoding.UTF8, "application/json"); var request = new HttpRequestMessage(HttpMethod.Post, url) { Content = content }; var response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, CancelToken.Token); if (response.IsSuccessStatusCode == false) { var responseString = await response.Content.ReadAsStringAsync(); throw new InvalidOperationException($"Failed to export database from server: {ServerUrl}, " + $"status code: {response.StatusCode}, " + $"error: {responseString}"); } using (var responseStream = await response.Content.ReadAsStreamAsync()) using (var stream = new GZipStream(responseStream, mode: CompressionMode.Decompress)) using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var source = new StreamSource(stream, context)) { var destination = new DatabaseDestination(Database); var options = new DatabaseSmugglerOptionsServerSide(); var smuggler = new Documents.DatabaseSmuggler(Database, source, destination, Database.Time, options, Result, OnProgress, CancelToken.Token); smuggler.Execute(); } }
public async Task Documents() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var stream = new ArrayStream(RequestBodyStream(), "Docs")) using (var source = new StreamSource(stream, context, Database)) { var destination = new DatabaseDestination(Database); var options = new DatabaseSmugglerOptionsServerSide { ReadLegacyEtag = true, OperateOnTypes = DatabaseItemType.Documents }; var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options); var result = smuggler.Execute(); var replicationSource = GetSourceReplicationInformation(context, GetRemoteServerInstanceId(), out var documentId); replicationSource.LastDocumentEtag = result.LegacyLastDocumentEtag; replicationSource.Source = GetFromServer(); replicationSource.LastBatchSize = result.Documents.ReadCount + result.Tombstones.ReadCount; replicationSource.LastModified = DateTime.UtcNow; await SaveSourceReplicationInformation(replicationSource, context, documentId); } }
private async Task MigrateIndexes() { var response = await RunWithAuthRetry(async() => { var url = $"{Options.ServerUrl}/databases/{Options.DatabaseName}/indexes"; var request = new HttpRequestMessage(HttpMethod.Get, url); var responseMessage = await Parameters.HttpClient.SendAsync(request, Parameters.CancelToken.Token); return(responseMessage); }); if (response.IsSuccessStatusCode == false) { var responseString = await response.Content.ReadAsStringAsync(); throw new InvalidOperationException($"Failed to export indexes from server: {Options.ServerUrl}, " + $"status code: {response.StatusCode}, " + $"error: {responseString}"); } using (var responseStream = await response.Content.ReadAsStreamAsync()) using (var indexesStream = new ArrayStream(responseStream, "Indexes")) // indexes endpoint returns an array using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var source = new StreamSource(indexesStream, context, Parameters.Database)) { var destination = new DatabaseDestination(Parameters.Database); var options = new DatabaseSmugglerOptionsServerSide { RemoveAnalyzers = Options.RemoveAnalyzers, }; var smuggler = new DatabaseSmuggler(Parameters.Database, source, destination, Parameters.Database.Time, options, Parameters.Result, Parameters.OnProgress, Parameters.CancelToken.Token); smuggler.Execute(); } }
private void ApplyBackwardCompatibility(DatabaseSmugglerOptionsServerSide options) { if (options == null) { return; } if (RequestRouter.TryGetClientVersion(HttpContext, out var version) == false) { return; } if (version.Major != RavenVersionAttribute.Instance.MajorVersion) { return; } // only all 4.0 and 4.1 less or equal to 41006 if (version.Revision < 50 || version.Revision > 41006) { return; } if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents)) { options.OperateOnTypes |= DatabaseItemType.Attachments; } }
private async Task MigrateIndexes() { var url = $"{ServerUrl}/databases/{DatabaseName}/indexes"; var request = new HttpRequestMessage(HttpMethod.Get, url); var response = await _client.SendAsync(request, CancelToken.Token); if (response.IsSuccessStatusCode == false) { var responseString = await response.Content.ReadAsStringAsync(); throw new InvalidOperationException($"Failed to export indexes from server: {ServerUrl}, " + $"status code: {response.StatusCode}, " + $"error: {responseString}"); } using (var responseStream = await response.Content.ReadAsStreamAsync()) // indexes endpoint returns an array using (var indexesStream = new IndexesStream(responseStream)) using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var source = new StreamSource(indexesStream, context, Database)) { var destination = new DatabaseDestination(Database); var options = new DatabaseSmugglerOptionsServerSide(); var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, Result, OnProgress, CancelToken.Token); smuggler.Execute(); } }
private async Task MigrateDocuments(string lastEtag) { var url = $"{ServerUrl}/databases/{DatabaseName}/streams/docs?etag={lastEtag}"; var request = new HttpRequestMessage(HttpMethod.Get, url); var response = await HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, CancelToken.Token); if (response.IsSuccessStatusCode == false) { var responseString = await response.Content.ReadAsStringAsync(); throw new InvalidOperationException($"Failed to export documents from server: {ServerUrl}, " + $"status code: {response.StatusCode}, " + $"error: {responseString}"); } using (var responseStream = await response.Content.ReadAsStreamAsync()) using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var source = new StreamSource(responseStream, context, Database)) { var destination = new DatabaseDestination(Database); var options = new DatabaseSmugglerOptionsServerSide { ReadLegacyEtag = true }; var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, Result, OnProgress, CancelToken.Token); // since we will be migrating indexes as separate task don't ensureStepsProcessed at this point smuggler.Execute(ensureStepsProcessed: false); } }
/// <summary> /// Restore CompareExchange, Identities and Subscriptions from smuggler file when restoring snapshot. /// </summary> /// <param name="onProgress"></param> /// <param name="database"></param> /// <param name="smugglerFile"></param> /// <param name="context"></param> protected async Task RestoreFromSmugglerFile(Action <IOperationProgress> onProgress, DocumentDatabase database, string smugglerFile, DocumentsOperationContext context) { var destination = new DatabaseDestination(database); var smugglerOptions = new DatabaseSmugglerOptionsServerSide { AuthorizationStatus = AuthorizationStatus.DatabaseAdmin, OperateOnTypes = DatabaseItemType.CompareExchange | DatabaseItemType.Identities | DatabaseItemType.Subscriptions, SkipRevisionCreation = true }; var lastPath = GetSmugglerBackupPath(smugglerFile); using (var zip = await GetZipArchiveForSnapshot(lastPath)) { foreach (var entry in zip.Entries) { if (entry.Name == RestoreSettings.SmugglerValuesFileName) { using (var input = entry.Open()) using (var inputStream = GetSnapshotInputStream(input, database.Name)) using (var uncompressed = new GZipStream(inputStream, CompressionMode.Decompress)) { var source = new StreamSource(uncompressed, context, database); var smuggler = new Smuggler.Documents.DatabaseSmuggler(database, source, destination, database.Time, smugglerOptions, onProgress: onProgress, token: _operationCancelToken.Token); smuggler.Execute(ensureStepsProcessed: true, isLastFile: true); } break; } } } }
private void CreateBackup( DatabaseSmugglerOptionsServerSide options, string backupFilePath, long?startDocumentEtag, DocumentsOperationContext context, Action <IOperationProgress> onProgress) { // the last etag is already included in the last backup startDocumentEtag = startDocumentEtag == null ? 0 : ++startDocumentEtag; using (var file = File.Open(backupFilePath, FileMode.CreateNew)) { var smugglerSource = new DatabaseSource(_database, startDocumentEtag.Value); var smugglerDestination = new StreamDestination(file, context, smugglerSource); var smuggler = new Smuggler.Documents.DatabaseSmuggler(_database, smugglerSource, smugglerDestination, _database.Time, options: options, result: _backupResult, onProgress: onProgress, token: TaskCancelToken.Token); smuggler.Execute(); file.Flush(flushToDisk: true); } }
public StreamDocumentActions(BlittableJsonTextWriter writer, DocumentsOperationContext context, DatabaseSource source, DatabaseSmugglerOptionsServerSide options, Func <LazyStringValue, bool> filterMetadataProperty, string propertyName) : base(writer, propertyName) { _context = context; _source = source; _options = options; _filterMetadataProperty = filterMetadataProperty; }
public async Task PostExport() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { var operationId = GetLongQueryString("operationId", false) ?? Database.Operations.GetNextOperationId(); var startDocumentEtag = GetLongQueryString("startEtag", false) ?? 0; var stream = TryGetRequestFromStream("DownloadOptions") ?? RequestBodyStream(); DatabaseSmugglerOptionsServerSide options; using (context.GetManagedBuffer(out var buffer)) { var firstRead = await stream.ReadAsync(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Buffer.Count); buffer.Used = 0; buffer.Valid = firstRead; if (firstRead != 0) { var blittableJson = await context.ParseToMemoryAsync(stream, "DownloadOptions", BlittableJsonDocumentBuilder.UsageMode.None, buffer); options = JsonDeserializationServer.DatabaseSmugglerOptions(blittableJson); } else { // no content, we'll use defaults options = new DatabaseSmugglerOptionsServerSide(); } } ApplyBackwardCompatibility(options); var token = CreateOperationToken(); var fileName = options.FileName; if (string.IsNullOrEmpty(fileName)) { fileName = $"Dump of {context.DocumentDatabase.Name} {SystemTime.UtcNow.ToString("yyyy-MM-dd HH-mm", CultureInfo.InvariantCulture)}"; } var contentDisposition = "attachment; filename=" + Uri.EscapeDataString(fileName) + ".ravendbdump"; HttpContext.Response.Headers["Content-Disposition"] = contentDisposition; try { await Database.Operations.AddOperation( Database, "Export database: " + Database.Name, Operations.OperationType.DatabaseExport, onProgress => Task.Run(() => ExportDatabaseInternal(options, startDocumentEtag, onProgress, context, token), token.Token), operationId, token : token); } catch (Exception) { HttpContext.Abort(); } } }
private Stream GetInputStream(Stream fileStream, DatabaseSmugglerOptionsServerSide options) { if (options.EncryptionKey != null) { return(new DecryptingXChaCha20Oly1305Stream(fileStream, Convert.FromBase64String(options.EncryptionKey))); } return(fileStream); }
private Stream GetOutputStream(Stream fileStream, DatabaseSmugglerOptionsServerSide options) { if (options.EncryptionKey == null) { return(fileStream); } var key = options?.EncryptionKey; return(new EncryptingXChaCha20Poly1305Stream(fileStream, Convert.FromBase64String(key))); }
private async Task MigrateDatabase(long operationId, ImportInfo importInfo) { var startDocumentEtag = importInfo?.LastEtag ?? 0; var url = $"{Options.ServerUrl}/databases/{Options.DatabaseName}/smuggler/export?operationId={operationId}&startEtag={startDocumentEtag}"; var databaseSmugglerOptionsServerSide = new DatabaseSmugglerOptionsServerSide { OperateOnTypes = Options.OperateOnTypes, RemoveAnalyzers = Options.RemoveAnalyzers }; if (importInfo != null) { databaseSmugglerOptionsServerSide.OperateOnTypes |= DatabaseItemType.Tombstones; databaseSmugglerOptionsServerSide.OperateOnTypes |= DatabaseItemType.CompareExchangeTombstones; } var json = JsonConvert.SerializeObject(databaseSmugglerOptionsServerSide); var content = new StringContent(json, Encoding.UTF8, "application/json"); var request = new HttpRequestMessage(HttpMethod.Post, url) { Content = content }; var response = await Parameters.HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, Parameters.CancelToken.Token); if (response.IsSuccessStatusCode == false) { var responseString = await response.Content.ReadAsStringAsync(); throw new InvalidOperationException($"Failed to export database from server: {Options.ServerUrl}, " + $"status code: {response.StatusCode}, " + $"error: {responseString}"); } await using (var responseStream = await response.Content.ReadAsStreamAsync()) await using (var stream = new GZipStream(responseStream, mode: CompressionMode.Decompress)) using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var source = new StreamSource(stream, context, Parameters.Database)) { var destination = new DatabaseDestination(Parameters.Database); var options = new DatabaseSmugglerOptionsServerSide { TransformScript = Options.TransformScript, OperateOnTypes = Options.OperateOnTypes, OperateOnDatabaseRecordTypes = Options.OperateOnDatabaseRecordTypes }; var smuggler = new Documents.DatabaseSmuggler(Parameters.Database, source, destination, Parameters.Database.Time, options, Parameters.Result, Parameters.OnProgress, Parameters.CancelToken.Token); await smuggler.ExecuteAsync(); } }
public Task <SmugglerInitializeResult> InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result) { _reader = new StreamReader(_stream); _csvReader = new CsvReader(_reader, _csvHelperConfig); _result = result; var disposable = new DisposableAction(() => { _reader.Dispose(); _csvReader.Dispose(); }); return(Task.FromResult(new SmugglerInitializeResult(disposable, ServerVersion.DevBuildNumber))); }
private IOperationResult ExportDatabaseInternal( DatabaseSmugglerOptionsServerSide options, long startDocumentEtag, Action <IOperationProgress> onProgress, DocumentsOperationContext context, OperationCancelToken token) { using (token) { var source = new DatabaseSource(Database, startDocumentEtag); var destination = new StreamDestination(ResponseBodyStream(), context, source); var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, onProgress: onProgress, token: token.Token); return(smuggler.Execute()); } }
public IDisposable Initialize(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, out long buildVersion) { buildVersion = ServerVersion.DevBuildNumber; _reader = new StreamReader(_stream); _csvReader = new CsvReader(_reader, _csvHelperConfig); _result = result; return(new DisposableAction(() => { _reader.Dispose(); _csvReader.Dispose(); })); }
private void HandleLegacyIndexes() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var stream = new ArrayStream(RequestBodyStream(), nameof(DatabaseItemType.Indexes))) using (var source = new StreamSource(stream, context, Database)) { var destination = new DatabaseDestination(Database); var options = new DatabaseSmugglerOptionsServerSide { OperateOnTypes = DatabaseItemType.Indexes }; var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options); smuggler.Execute(); } }
private InternalBackupResult CreateBackup( DatabaseSmugglerOptionsServerSide options, string backupFilePath, long?startDocumentEtag, long?startRaftIndex) { // the last etag is already included in the last backup var currentBackupResults = new InternalBackupResult(); startDocumentEtag = startDocumentEtag == null ? 0 : ++startDocumentEtag; startRaftIndex = startRaftIndex == null ? 0 : ++startRaftIndex; using (Stream fileStream = File.Open(backupFilePath, FileMode.CreateNew)) using (var outputStream = GetOutputStream(fileStream)) using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { var smugglerSource = new DatabaseSource(_database, startDocumentEtag.Value, startRaftIndex.Value, _logger); var smugglerDestination = new StreamDestination(outputStream, context, smugglerSource); var smuggler = new DatabaseSmuggler(_database, smugglerSource, smugglerDestination, _database.Time, options: options, result: _backupResult, onProgress: _onProgress, token: TaskCancelToken.Token); smuggler.ExecuteAsync().Wait(); switch (outputStream) { case EncryptingXChaCha20Poly1305Stream encryptedStream: encryptedStream.Flush(flushToDisk: true); break; case FileStream file: file.Flush(flushToDisk: true); break; default: throw new InvalidOperationException($" {outputStream.GetType()} not supported"); } currentBackupResults.LastDocumentEtag = smugglerSource.LastEtag; currentBackupResults.LastDatabaseChangeVector = smugglerSource.LastDatabaseChangeVector; currentBackupResults.LastRaftIndex = smugglerSource.LastRaftIndex; return(currentBackupResults); } }
private async Task <SmugglerResult> MigrateDatabase(string json, bool readLegacyEtag) { var response = await RunWithAuthRetry(async() => { var url = $"{Options.ServerUrl}/databases/{Options.DatabaseName}/studio-tasks/exportDatabase"; var content = new StringContent(json, Encoding.UTF8, "application/json"); var request = new HttpRequestMessage(HttpMethod.Post, url) { Content = content }; var responseMessage = await Parameters.HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, Parameters.CancelToken.Token); return(responseMessage); }); if (response.IsSuccessStatusCode == false) { var responseString = await response.Content.ReadAsStringAsync(); throw new InvalidOperationException($"Failed to export database from server: {Options.ServerUrl}, " + $"status code: {response.StatusCode}, " + $"error: {responseString}"); } await using (var responseStream = await response.Content.ReadAsStreamAsync()) await using (var stream = new GZipStream(responseStream, mode: CompressionMode.Decompress)) using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var source = new StreamSource(stream, context, Parameters.Database)) { var destination = new DatabaseDestination(Parameters.Database); var options = new DatabaseSmugglerOptionsServerSide { #pragma warning disable 618 ReadLegacyEtag = readLegacyEtag, #pragma warning restore 618 RemoveAnalyzers = Options.RemoveAnalyzers, TransformScript = Options.TransformScript, OperateOnTypes = Options.OperateOnTypes }; var smuggler = new DatabaseSmuggler(Parameters.Database, source, destination, Parameters.Database.Time, options, Parameters.Result, Parameters.OnProgress, Parameters.CancelToken.Token); return(await smuggler.ExecuteAsync()); } }
public IDisposable Initialize(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, long buildVersion) { _gzipStream = new GZipStream(_stream, CompressionMode.Compress, leaveOpen: true); _writer = new BlittableJsonTextWriter(_context, _gzipStream); _options = options; _writer.WriteStartObject(); _writer.WritePropertyName("BuildVersion"); _writer.WriteInteger(buildVersion); return(new DisposableAction(() => { _writer.WriteEndObject(); _writer.Dispose(); _gzipStream.Dispose(); })); }
private void RestoreFromLastFile(Action <IOperationProgress> onProgress, DocumentDatabase database, string lastFile, DocumentsOperationContext context, RestoreResult result) { var destination = new DatabaseDestination(database); var smugglerOptions = new DatabaseSmugglerOptionsServerSide { AuthorizationStatus = AuthorizationStatus.DatabaseAdmin, OperateOnTypes = DatabaseItemType.CompareExchange | DatabaseItemType.Identities, SkipRevisionCreation = true, KeepOriginalChangeVector = true }; var lastPath = Path.Combine(_restoreConfiguration.BackupLocation, lastFile); if (Path.GetExtension(lastPath) == Constants.Documents.PeriodicBackup.SnapshotExtension) { using (var zip = ZipFile.Open(lastPath, ZipArchiveMode.Read, System.Text.Encoding.UTF8)) { foreach (var entry in zip.Entries) { if (entry.Name == RestoreSettings.SmugglerValuesFileName) { using (var input = entry.Open()) using (var uncompressed = new GZipStream(input, CompressionMode.Decompress)) { var source = new StreamSource(uncompressed, context, database); var smuggler = new Smuggler.Documents.DatabaseSmuggler(database, source, destination, database.Time, smugglerOptions, onProgress: onProgress, token: _operationCancelToken.Token); smuggler.Execute(); } break; } } } } else { ImportSingleBackupFile(database, onProgress, null, lastPath, context, destination, smugglerOptions); } result.Identities.Processed = true; result.CompareExchange.Processed = true; onProgress.Invoke(result.Progress); }
private async Task <IOperationResult> ExportDatabaseInternalAsync( DatabaseSmugglerOptionsServerSide options, long startDocumentEtag, long startRaftIndex, Action <IOperationProgress> onProgress, DocumentsOperationContext context, OperationCancelToken token) { using (token) { var source = new DatabaseSource(Database, startDocumentEtag, startRaftIndex, Logger); await using (var outputStream = GetOutputStream(ResponseBodyStream(), options)) { var destination = new StreamDestination(outputStream, context, source); var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, onProgress: onProgress, token: token.Token); return(await smuggler.ExecuteAsync()); } } }
private void ImportSingleBackupFile(DocumentDatabase database, Action <IOperationProgress> onProgress, RestoreResult restoreResult, string filePath, DocumentsOperationContext context, DatabaseDestination destination, DatabaseSmugglerOptionsServerSide options, Action <IndexDefinitionAndType> onIndexAction = null) { using (var fileStream = File.Open(filePath, FileMode.Open)) using (var stream = new GZipStream(new BufferedStream(fileStream, 128 * Voron.Global.Constants.Size.Kilobyte), CompressionMode.Decompress)) using (var source = new StreamSource(stream, context, database)) { var smuggler = new Smuggler.Documents.DatabaseSmuggler(database, source, destination, database.Time, options, result: restoreResult, onProgress: onProgress, token: _operationCancelToken.Token) { OnIndexAction = onIndexAction, }; smuggler.Execute(); } }
public Task <SmugglerInitializeResult> InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result) { _currentTypeIndex = 0; if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) || options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments) || options.OperateOnTypes.HasFlag(DatabaseItemType.Tombstones) || options.OperateOnTypes.HasFlag(DatabaseItemType.Conflicts) || options.OperateOnTypes.HasFlag(DatabaseItemType.CounterGroups) || options.OperateOnTypes.HasFlag(DatabaseItemType.TimeSeries)) { _returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context); _disposeTransaction = _context.OpenReadTransaction(); LastEtag = DocumentsStorage.ReadLastEtag(_disposeTransaction.InnerTransaction); LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(_disposeTransaction.InnerTransaction); } if (options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchange) || options.OperateOnTypes.HasFlag(DatabaseItemType.Identities) || options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchangeTombstones) || options.OperateOnTypes.HasFlag(DatabaseItemType.Subscriptions) || options.OperateOnTypes.HasFlag(DatabaseItemType.ReplicationHubCertificates)) { _returnServerContext = _database.ServerStore.ContextPool.AllocateOperationContext(out _serverContext); _disposeServerTransaction = _serverContext.OpenReadTransaction(); using (var rawRecord = _database.ServerStore.Cluster.ReadRawDatabaseRecord(_serverContext, _database.Name)) { LastRaftIndex = rawRecord.EtagForBackup; } } var disposable = new DisposableAction(() => { _disposeServerTransaction?.Dispose(); _returnServerContext?.Dispose(); _disposeTransaction?.Dispose(); _returnContext?.Dispose(); }); return(Task.FromResult(new SmugglerInitializeResult(disposable, ServerVersion.Build))); }
public DatabaseSmuggler(DocumentDatabase database, ISmugglerSource source, ISmugglerDestination destination, SystemTime time, DatabaseSmugglerOptionsServerSide options = null, SmugglerResult result = null, Action <IOperationProgress> onProgress = null, CancellationToken token = default) { _database = database; _source = source; _destination = destination; _options = options ?? new DatabaseSmugglerOptionsServerSide(); _result = result; _token = token; if (string.IsNullOrWhiteSpace(_options.TransformScript) == false) { _patcher = new SmugglerPatcher(_options, database); } _time = time; _onProgress = onProgress ?? (progress => { }); }
public IAsyncDisposable InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, long buildVersion) { _gzipStream = new GZipStream(_stream, CompressionMode.Compress, leaveOpen: true); _writer = new AsyncBlittableJsonTextWriter(_context, _gzipStream); _options = options; SetupMetadataFilterMethod(_context); _writer.WriteStartObject(); _writer.WritePropertyName("BuildVersion"); _writer.WriteInteger(buildVersion); return(new AsyncDisposableAction(async() => { _writer.WriteEndObject(); await _writer.DisposeAsync(); await _gzipStream.DisposeAsync(); })); }
public async Task PostImport() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { var options = DatabaseSmugglerOptionsServerSide.Create(HttpContext, context); using (var stream = new GZipStream(new BufferedStream(await GetImportStream(), 128 * Voron.Global.Constants.Size.Kilobyte), CompressionMode.Decompress)) using (var token = CreateOperationToken()) using (var source = new StreamSource(stream, context)) { var destination = new DatabaseDestination(Database); var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, token: token.Token); var result = smuggler.Execute(); WriteImportResult(context, result, ResponseBodyStream()); } } }
private void ApplyBackwardCompatibility(DatabaseSmugglerOptionsServerSide options) { if (options == null) { return; } if (((options.OperateOnTypes & DatabaseItemType.DatabaseRecord) != 0) && (options.OperateOnDatabaseRecordTypes == DatabaseRecordItemType.None)) { options.OperateOnDatabaseRecordTypes = DatabaseSmugglerOptions.DefaultOperateOnDatabaseRecordTypes; } if (RequestRouter.TryGetClientVersion(HttpContext, out var version) == false) { return; } if (version.Major != RavenVersionAttribute.Instance.MajorVersion) { return; } #pragma warning disable 618 if (version.Minor < 2 && options.OperateOnTypes.HasFlag(DatabaseItemType.Counters)) #pragma warning restore 618 { options.OperateOnTypes |= DatabaseItemType.CounterGroups; } // only all 4.0 and 4.1 less or equal to 41006 if (version.Revision < 50 || version.Revision > 41006) { return; } if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents)) { options.OperateOnTypes |= DatabaseItemType.Attachments; } }
private async Task ImportSingleBackupFile(DocumentDatabase database, Action <IOperationProgress> onProgress, RestoreResult restoreResult, string filePath, DocumentsOperationContext context, DatabaseDestination destination, DatabaseSmugglerOptionsServerSide options, bool isLastFile, Action <IndexDefinitionAndType> onIndexAction = null, Action <DatabaseRecord> onDatabaseRecordAction = null) { using (var fileStream = await GetStream(filePath)) using (var inputStream = GetInputStream(fileStream, database.MasterKey)) using (var gzipStream = new GZipStream(inputStream, CompressionMode.Decompress)) using (var source = new StreamSource(gzipStream, context, database)) { var smuggler = new Smuggler.Documents.DatabaseSmuggler(database, source, destination, database.Time, options, result: restoreResult, onProgress: onProgress, token: _operationCancelToken.Token) { OnIndexAction = onIndexAction, OnDatabaseRecordAction = onDatabaseRecordAction }; smuggler.Execute(ensureStepsProcessed: false, isLastFile); } }