public void Northwind_RemoteToFile_FileToRemote_Test() { using (var store1 = CreateStore()) using (var store2 = CreateStore()) { DeployNorthwind(store1); WaitForIndexing(store1); var outputFile = Path.Combine(NewDataPath(forceCreateDir: true), "backup.ravendump"); var smuggler = new DatabaseSmuggler(new DatabaseSmugglerOptions(), new DatabaseSmugglerRemoteSource(store1), new DatabaseSmugglerFileDestination(outputFile)); smuggler.Execute(); smuggler = new DatabaseSmuggler(new DatabaseSmugglerOptions(), new DatabaseSmugglerFileSource(outputFile), new DatabaseSmugglerRemoteDestination(store2)); smuggler.Execute(); WaitForIndexing(store2); var statistics = store2.DatabaseCommands.GetStatistics(); Assert.Equal(1059, statistics.CountOfDocuments); Assert.Equal(4, statistics.CountOfIndexes); Assert.Equal(1, statistics.CountOfResultTransformers); } }
public async Task GetImport() { if (HttpContext.Request.Query.ContainsKey("file") == false && HttpContext.Request.Query.ContainsKey("url") == false) { throw new ArgumentException("'file' or 'url' are mandatory when using GET /smuggler/import"); } using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { var options = DatabaseSmugglerOptionsServerSide.Create(HttpContext); using (var stream = new GZipStream(new BufferedStream(await GetImportStream(), 128 * Voron.Global.Constants.Size.Kilobyte), CompressionMode.Decompress)) using (var token = CreateOperationToken()) using (var source = new StreamSource(stream, context, Database)) { var destination = new DatabaseDestination(Database); var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, token: token.Token); var result = smuggler.Execute(); WriteImportResult(context, result, ResponseBodyStream()); } } }
private static void DeployNorthwindAndExportToFile(DocumentStore store, string path) { DeployNorthwind(store); var smuggler = new DatabaseSmuggler(new DatabaseSmugglerOptions(), new DatabaseSmugglerRemoteSource(store), new DatabaseSmugglerFileDestination(path)); smuggler.Execute(); }
private IOperationResult ExportDatabaseInternal(DatabaseSmugglerOptions options, Action <IOperationProgress> onProgress, DocumentsOperationContext context, OperationCancelToken token) { using (token) { var source = new DatabaseSource(Database, 0); var destination = new StreamDestination(ResponseBodyStream(), context, source); var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, onProgress: onProgress, token: token.Token); return(smuggler.Execute()); } }
private void DoImportInternal(DocumentsOperationContext context, Stream stream, DatabaseSmugglerOptionsServerSide options, SmugglerResult result, Action<IOperationProgress> onProgress, OperationCancelToken token) { using (stream) using (token) using (var source = new StreamSource(stream, context, Database)) { var destination = new DatabaseDestination(Database); var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, result, onProgress, token.Token); smuggler.Execute(); } }
private void ImportDocumentsFromCsvStream(Stream stream, DocumentsOperationContext context, string entity, DatabaseSmugglerOptionsServerSide options, SmugglerResult result, Action<IOperationProgress> onProgress, OperationCancelToken token) { if (string.IsNullOrEmpty(entity) == false && char.IsLower(entity[0])) entity = char.ToUpper(entity[0]) + entity.Substring(1); result.AddInfo($"Import collection: {entity}"); using (var source = new CsvStreamSource(Database, stream, context, entity)) { var destination = new DatabaseDestination(Database); var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, result, onProgress, token.Token); smuggler.Execute(); } }
public void ShouldSmuggleOnlyTransformers() { var destination = new DatabaseSmugglerCountingDestination(); var smuggler = new DatabaseSmuggler(new DatabaseSmugglerOptions { OperateOnTypes = DatabaseItemType.Transformers }, new DatabaseSmugglerFileSource(_path), destination); smuggler.Execute(); Assert.Equal(0, destination.WroteDocuments); Assert.Equal(0, destination.WroteDocumentDeletions); Assert.Equal(0, destination.WroteIdentities); Assert.Equal(0, destination.WroteIndexes); Assert.Equal(1, destination.WroteTransformers); }
/// <summary> /// Initializes this instance. /// </summary> /// <returns></returns> public IDocumentStore Initialize(bool ensureDatabaseExists) { if (initialized) { return(this); } AssertValidConfiguration(); jsonRequestFactory = new HttpJsonRequestFactory(MaxNumberOfCachedRequests, HttpMessageHandlerFactory, Conventions.AcceptGzipContent, Conventions.AuthenticationScheme); try { if (string.IsNullOrEmpty(ApiKey) == false) { Credentials = null; } SecurityExtensions.InitializeSecurity(Conventions, jsonRequestFactory, Url, Credentials); InitializeInternal(); if (Conventions.DocumentKeyGenerator == null)// don't overwrite what the user is doing { var generator = new MultiDatabaseHiLoGenerator(32); Conventions.DocumentKeyGenerator = (dbName, databaseCommands, entity) => generator.GenerateDocumentKey(dbName, databaseCommands, Conventions, entity); } if (Conventions.AsyncDocumentKeyGenerator == null && asyncDatabaseCommandsGenerator != null) { var generator = new AsyncMultiDatabaseHiLoKeyGenerator(32); Conventions.AsyncDocumentKeyGenerator = (dbName, commands, entity) => generator.GenerateDocumentKeyAsync(dbName, commands, Conventions, entity); } Smuggler = new DatabaseSmuggler(this); initialized = true; } catch (Exception) { Dispose(); throw; } return(this); }
private IOperationResult ExportDatabaseInternal( DatabaseSmugglerOptionsServerSide options, long startDocumentEtag, long startRaftIndex, Action <IOperationProgress> onProgress, DocumentsOperationContext context, OperationCancelToken token) { using (token) { var source = new DatabaseSource(Database, startDocumentEtag, startRaftIndex); using (var outputStream = GetOutputStream(ResponseBodyStream(), options)) { var destination = new StreamDestination(outputStream, context, source); var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, onProgress: onProgress, token: token.Token); return(smuggler.Execute()); } } }
public async Task PostImport() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { var options = DatabaseSmugglerOptionsServerSide.Create(HttpContext, context); using (var stream = new GZipStream(new BufferedStream(await GetImportStream(), 128 * Voron.Global.Constants.Size.Kilobyte), CompressionMode.Decompress)) using (var token = CreateOperationToken()) using (var source = new StreamSource(stream, context)) { var destination = new DatabaseDestination(Database); var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, token: token.Token); var result = smuggler.Execute(); WriteImportResult(context, result, ResponseBodyStream()); } } }
public void ShouldSmuggleOnlyTransformers() { using (var store = NewRemoteDocumentStore()) { DeployNorthwind(store); var destination = new DatabaseSmugglerCountingDestination(); var smuggler = new DatabaseSmuggler(new DatabaseSmugglerOptions { OperateOnTypes = DatabaseItemType.Transformers }, new DatabaseSmugglerRemoteSource(store), destination); smuggler.Execute(); Assert.Equal(0, destination.WroteDocuments); Assert.Equal(0, destination.WroteDocumentDeletions); Assert.Equal(0, destination.WroteIdentities); Assert.Equal(0, destination.WroteIndexes); Assert.Equal(1, destination.WroteTransformers); } }
public void ShouldSmuggleOnlyTransformers() { using (var store = NewRemoteDocumentStore()) { var database = servers[0].Options.DatabaseLandlord.GetResourceInternal(store.DefaultDatabase).Result; DeployNorthwind(store); var destination = new DatabaseSmugglerCountingDestination(); var smuggler = new DatabaseSmuggler(new DatabaseSmugglerOptions { OperateOnTypes = DatabaseItemType.Transformers }, new DatabaseSmugglerEmbeddedSource(database), destination); smuggler.Execute(); Assert.Equal(0, destination.WroteDocuments); Assert.Equal(0, destination.WroteDocumentDeletions); Assert.Equal(0, destination.WroteIdentities); Assert.Equal(0, destination.WroteIndexes); Assert.Equal(1, destination.WroteTransformers); } }
public Task <HttpResponseMessage> ExportDatabase([FromBody] ExportData exportData) { var requestString = exportData.SmugglerOptions; DatabaseSmugglerOptions smugglerOptions; using (var jsonReader = new RavenJsonTextReader(new StringReader(requestString))) { var serializer = JsonExtensions.CreateDefaultJsonSerializer(); smugglerOptions = (DatabaseSmugglerOptions)serializer.Deserialize(jsonReader, typeof(DatabaseSmugglerOptions)); } var result = GetEmptyMessage(); // create PushStreamContent object that will be called when the output stream will be ready. result.Content = new PushStreamContent(async(outputStream, content, arg3) => { try { var smuggler = new DatabaseSmuggler(smugglerOptions, new DatabaseSmugglerEmbeddedSource(Database), new DatabaseSmugglerStreamDestination(outputStream, leaveOpen: true)); await smuggler.ExecuteAsync().ConfigureAwait(false); } finally { outputStream.Close(); } }); var fileName = string.IsNullOrEmpty(exportData.FileName) || (exportData.FileName.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0) ? string.Format("Dump of {0}, {1}", DatabaseName, DateTime.Now.ToString("yyyy-MM-dd HH-mm")) : exportData.FileName; result.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment") { FileName = fileName + ".ravendump" }; return(new CompletedTask <HttpResponseMessage>(result)); }
public async Task Alias_in_edge_relationships_select_clause_should_work() { using (var store = GetDocumentStore()) { var smuggler = new DatabaseSmuggler(store); var op = await smuggler.ImportAsync(new DatabaseSmugglerImportOptions(), "./Issues/RavenDB-13499.ravendbdump"); var _ = await op.WaitForCompletionAsync(TimeSpan.FromMinutes(5)); using (var session = store.OpenSession()) { var results1 = session.Advanced .GraphQuery <JObject>("match (Person as f)-[Relationships as r select TargetId]->(Person as t)") .ToArray(); var results2 = session.Advanced .GraphQuery <JObject>("match (Person as f)-[Relationships as r select r.TargetId]->(Person as t)") .ToArray(); Assert.Equal(results1, results2); } } }
public async Task <HttpResponseMessage> CreateSampleData() { var results = Database.Queries.Query(Constants.DocumentsByEntityNameIndex, new IndexQuery(), CancellationToken.None); if (results.Results.Count > 0) { return(GetMessageWithString("You cannot create sample data in a database that already contains documents", HttpStatusCode.BadRequest)); } using (var stream = typeof(StudioTasksController).Assembly.GetManifestResourceStream("Raven.Database.Server.Assets.EmbeddedData.Northwind.dump")) { var smuggler = new DatabaseSmuggler(new DatabaseSmugglerOptions { OperateOnTypes = DatabaseItemType.Documents | DatabaseItemType.Indexes | DatabaseItemType.Transformers, ShouldExcludeExpired = false }, new DatabaseSmugglerStreamSource(stream), new DatabaseSmugglerEmbeddedDestination(Database)); await smuggler.ExecuteAsync().ConfigureAwait(false); } return(GetEmptyMessage()); }
private async Task BulkImport(BlockingCollection <Func <Task <Stream> > > files, string directory) { var results = new ConcurrentQueue <SmugglerResult>(); var tasks = new Task[Math.Max(1, ProcessorInfo.ProcessorCount / 2)]; var finalResult = new SmugglerResult(); for (int i = 0; i < tasks.Length; i++) { tasks[i] = Task.Run(async() => { while (files.IsCompleted == false) { Func <Task <Stream> > getFile; try { getFile = files.Take(); } catch (Exception) { //TODO : add logging, _silently_ skipping is a bad idea continue; } using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var file = await getFile()) using (var stream = new GZipStream(new BufferedStream(file, 128 * Voron.Global.Constants.Size.Kilobyte), CompressionMode.Decompress)) using (var source = new StreamSource(stream, context)) { var destination = new DatabaseDestination(Database); var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time); var result = smuggler.Execute(); results.Enqueue(result); } } }); } await Task.WhenAll(tasks); while (results.TryDequeue(out SmugglerResult importResult)) { finalResult.Documents.SkippedCount += importResult.Documents.SkippedCount; finalResult.Documents.ReadCount += importResult.Documents.ReadCount; finalResult.Documents.ErroredCount += importResult.Documents.ErroredCount; finalResult.Documents.LastEtag = Math.Max(finalResult.Documents.LastEtag, importResult.Documents.LastEtag); finalResult.Documents.Attachments = importResult.Documents.Attachments; finalResult.RevisionDocuments.ReadCount += importResult.RevisionDocuments.ReadCount; finalResult.RevisionDocuments.ErroredCount += importResult.RevisionDocuments.ErroredCount; finalResult.RevisionDocuments.LastEtag = Math.Max(finalResult.RevisionDocuments.LastEtag, importResult.RevisionDocuments.LastEtag); finalResult.RevisionDocuments.Attachments = importResult.RevisionDocuments.Attachments; finalResult.Identities.ReadCount += importResult.Identities.ReadCount; finalResult.Identities.ErroredCount += importResult.Identities.ErroredCount; finalResult.Indexes.ReadCount += importResult.Indexes.ReadCount; finalResult.Indexes.ErroredCount += importResult.Indexes.ErroredCount; foreach (var message in importResult.Messages) { finalResult.AddMessage(message); } } using (ContextPool.AllocateOperationContext(out DocumentsOperationContext finalContext)) { var memoryStream = new MemoryStream(); WriteImportResult(finalContext, finalResult, memoryStream); memoryStream.Position = 0; try { using (var output = File.Create(Path.Combine(directory, "smuggler.results.txt"))) { memoryStream.CopyTo(output); } } catch (Exception) { // ignore any failure here } memoryStream.Position = 0; memoryStream.CopyTo(ResponseBodyStream()); } }
public async Task <HttpResponseMessage> ImportDatabase(int batchSize, bool includeExpiredDocuments, bool stripReplicationInformation, bool shouldDisableVersioningBundle, DatabaseItemType operateOnTypes, string filtersPipeDelimited, string transformScript) { if (!Request.Content.IsMimeMultipartContent()) { throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType); } string tempPath = Database.Configuration.Core.TempPath; var fullTempPath = tempPath + Constants.TempUploadsDirectoryName; if (File.Exists(fullTempPath)) { File.Delete(fullTempPath); } if (Directory.Exists(fullTempPath) == false) { Directory.CreateDirectory(fullTempPath); } var streamProvider = new MultipartFileStreamProvider(fullTempPath); await Request.Content.ReadAsMultipartAsync(streamProvider).ConfigureAwait(false); var uploadedFilePath = streamProvider.FileData[0].LocalFileName; string fileName = null; var fileContent = streamProvider.Contents.SingleOrDefault(); if (fileContent != null) { fileName = fileContent.Headers.ContentDisposition.FileName.Replace("\"", string.Empty); } var status = new ImportOperationStatus(); var cts = new CancellationTokenSource(); var task = Task.Run(async() => { try { using (var fileStream = File.Open(uploadedFilePath, FileMode.Open, FileAccess.Read)) { var smugglerOptions = new DatabaseSmugglerOptions(); smugglerOptions.BatchSize = batchSize; smugglerOptions.ShouldExcludeExpired = !includeExpiredDocuments; smugglerOptions.StripReplicationInformation = stripReplicationInformation; smugglerOptions.ShouldDisableVersioningBundle = shouldDisableVersioningBundle; smugglerOptions.OperateOnTypes = operateOnTypes; smugglerOptions.TransformScript = transformScript; // Filters are passed in without the aid of the model binder. Instead, we pass in a list of FilterSettings using a string like this: pathHere;;;valueHere;;;true|||againPathHere;;;anotherValue;;;false // Why? Because I don't see a way to pass a list of a values to a WebAPI method that accepts a file upload, outside of passing in a simple string value and parsing it ourselves. if (filtersPipeDelimited != null) { smugglerOptions.Filters.AddRange(filtersPipeDelimited .Split(new[] { "|||" }, StringSplitOptions.RemoveEmptyEntries) .Select(f => f.Split(new[] { ";;;" }, StringSplitOptions.RemoveEmptyEntries)) .Select(o => new FilterSetting { Path = o[0], Values = new List <string> { o[1] }, ShouldMatch = bool.Parse(o[2]) })); } var smuggler = new DatabaseSmuggler(smugglerOptions, new DatabaseSmugglerStreamSource(fileStream), new DatabaseSmugglerEmbeddedDestination(Database)); smuggler.Notifications.OnProgress += (sender, message) => status.LastProgress = message; await smuggler.ExecuteAsync(cts.Token).ConfigureAwait(false); } } catch (Exception e) { status.Faulted = true; status.State = RavenJObject.FromObject(new { Error = e.ToString() }); if (cts.Token.IsCancellationRequested) { status.State = RavenJObject.FromObject(new { Error = "Task was cancelled" }); cts.Token.ThrowIfCancellationRequested(); //needed for displaying the task status as canceled and not faulted } if (e is InvalidDataException) { status.ExceptionDetails = e.Message; } else if (e is JsonReaderException) { status.ExceptionDetails = "Failed to load JSON Data. Please make sure you are importing .ravendump file, exported by smuggler (aka database export). If you are importing a .ravnedump file then the file may be corrupted"; } else if (e is OperationVetoedException && e.Message.Contains(VersioningPutTrigger.CreationOfHistoricalRevisionIsNotAllowed)) { status.ExceptionDetails = "You are trying to import historical documents while the versioning bundle is enabled. " + "The versioning bundle is enabled. You should disable versioning during import. " + "Please mark the checkbox 'Disable versioning bundle during import' at Import Database: Advanced settings before importing"; } else { status.ExceptionDetails = e.ToString(); } throw; } finally { status.Completed = true; File.Delete(uploadedFilePath); } }, cts.Token); long id; Database.Tasks.AddTask(task, status, new TaskActions.PendingTaskDescription { StartTime = SystemTime.UtcNow, TaskType = TaskActions.PendingTaskType.ImportDatabase, Payload = fileName, }, out id, cts); return(GetMessageWithObject(new { OperationId = id }, HttpStatusCode.Accepted)); }
private void TimerCallback(bool fullBackup) { if (currentTask != null) { return; } if (Database.Disposed) { Dispose(); return; } // we have shared lock for both incremental and full backup. lock (this) { if (currentTask != null) { return; } currentTask = Task.Factory.StartNew(async() => { var documentDatabase = Database; if (documentDatabase == null) { return; } using (LogContext.WithDatabase(documentDatabase.Name)) { try { DatabaseSmugglerOperationState exportResult; bool performAnotherRun = false; do { var localBackupConfigs = exportConfigs; var localBackupStatus = exportStatus; if (localBackupConfigs == null) { return; } if (localBackupConfigs.Disabled) { return; } var source = new DatabaseSmugglerEmbeddedSource(Database); if (fullBackup == false) { var currentEtags = await source.FetchCurrentMaxEtagsAsync(CancellationToken.None).ConfigureAwait(false); // No-op if nothing has changed if (currentEtags.LastDocsEtag == localBackupStatus.LastDocsEtag && currentEtags.LastDocDeleteEtag == localBackupStatus.LastDocsDeletionEtag) { return; } } var backupPath = localBackupConfigs.LocalFolderName ?? Path.Combine(documentDatabase.Configuration.Core.DataDirectory, "PeriodicExport-Temp"); if (Directory.Exists(backupPath) == false) { Directory.CreateDirectory(backupPath); } if (fullBackup) { // create filename for full dump backupPath = Path.Combine(backupPath, SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + ".ravendb-full-dump"); if (File.Exists(backupPath)) { var counter = 1; while (true) { backupPath = Path.Combine(Path.GetDirectoryName(backupPath), SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + " - " + counter + ".ravendb-full-dump"); if (File.Exists(backupPath) == false) { break; } counter++; } } } var smugglerOptions = new DatabaseSmugglerOptions { Limit = backupLimit }; var smugglerFileOptions = new DatabaseSmugglerFileDestinationOptions(); if (fullBackup == false) { smugglerOptions.StartDocsEtag = localBackupStatus.LastDocsEtag; smugglerOptions.StartDocsDeletionEtag = localBackupStatus.LastDocsDeletionEtag; smugglerFileOptions.Incremental = true; } var smuggler = new DatabaseSmuggler(smugglerOptions, source, new DatabaseSmugglerFileDestination(backupPath, smugglerFileOptions)); exportResult = await smuggler.ExecuteAsync().ConfigureAwait(false); if (fullBackup == false) { // No-op if nothing has changed if (exportResult.LastDocsEtag == localBackupStatus.LastDocsEtag && exportResult.LastDocDeleteEtag == localBackupStatus.LastDocsDeletionEtag) { logger.Info("Periodic export returned prematurely, nothing has changed since last export"); return; } } try { UploadToServer(exportResult.FilePath, localBackupConfigs, fullBackup); } finally { // if user did not specify local folder we delete temporary file. if (String.IsNullOrEmpty(localBackupConfigs.LocalFolderName)) { IOExtensions.DeleteFile(exportResult.FilePath); } } localBackupStatus.LastDocsEtag = exportResult.LastDocsEtag; localBackupStatus.LastDocsDeletionEtag = exportResult.LastDocDeleteEtag; if (fullBackup) { localBackupStatus.LastFullBackup = SystemTime.UtcNow; } else { localBackupStatus.LastBackup = SystemTime.UtcNow; } var ravenJObject = JsonExtensions.ToJObject(localBackupStatus); ravenJObject.Remove("Id"); var putResult = documentDatabase.Documents.Put(PeriodicExportStatus.RavenDocumentKey, null, ravenJObject, new RavenJObject(), null); // this result in exportStatus being refreshed localBackupStatus = exportStatus; if (localBackupStatus != null) { if (localBackupStatus.LastDocsEtag.IncrementBy(1) == putResult.ETag) // the last etag is with just us { localBackupStatus.LastDocsEtag = putResult.ETag; // so we can skip it for the next time } } if (backupLimit != int.MaxValue) { backupLimit = int.MaxValue; performAnotherRun = true; } else { performAnotherRun = false; } } while (performAnotherRun); } catch (ObjectDisposedException) { // shutting down, probably } catch (Exception e) { backupLimit = 100; logger.ErrorException("Error when performing periodic export", e); Database.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Message = e.Message, Title = "Error in Periodic Export", Exception = e.ToString(), UniqueKey = "Periodic Export Error", }); } } }) .Unwrap(); currentTask.ContinueWith(_ => { currentTask = null; }); } }