public async Task export_result_cannot_contain_neither_files_scheduled_for_deletion_nor_tombstones(string storage) { var exportStream = new MemoryStream(); using (var store = this.NewStore(requestedStorage: storage)) { using (var session = store.OpenAsyncSession()) { for (int i = 0; i < 10; i++) { session.RegisterUpload("prefix-" + i + ".file", CreateUniformFileStream(10)); } await session.SaveChangesAsync(); } using (var session = store.OpenAsyncSession()) { session.RegisterFileDeletion("prefix-3.file"); await session.SaveChangesAsync(); } var fs = GetFileSystem(); var exporter = new FilesystemDataDumper(fs); await exporter.ExportData( new SmugglerExportOptions <FilesConnectionStringOptions> { ToStream = exportStream }).ConfigureAwait(false); using (var import = NewStore(1)) { exportStream.Position = 0; var importedFs = GetFileSystem(1); var importer = new FilesystemDataDumper(importedFs); await importer.ImportData(new SmugglerImportOptions <FilesConnectionStringOptions> { FromStream = exportStream }); importedFs.Storage.Batch(accessor => { Assert.Equal(9, accessor.GetFilesAfter(Etag.Empty, 100).Count()); }); } } }
public Task <HttpResponseMessage> ExportFilesystem(StudioTasksController.ExportData smugglerOptionsJson) { var requestString = smugglerOptionsJson.SmugglerOptions; SmugglerFilesOptions smugglerOptions; using (var jsonReader = new RavenJsonTextReader(new StringReader(requestString))) { var serializer = JsonExtensions.CreateDefaultJsonSerializer(); smugglerOptions = (SmugglerFilesOptions)serializer.Deserialize(jsonReader, typeof(SmugglerFilesOptions)); } var result = GetEmptyMessage(); // create PushStreamContent object that will be called when the output stream will be ready. result.Content = new PushStreamContent(async(outputStream, content, arg3) => { try { var dataDumper = new FilesystemDataDumper(FileSystem, smugglerOptions); await dataDumper.ExportData( new SmugglerExportOptions <FilesConnectionStringOptions> { ToStream = outputStream }).ConfigureAwait(false); } finally { outputStream.Close(); } }); var fileName = String.IsNullOrEmpty(smugglerOptions.NoneDefualtFileName) || (smugglerOptions.NoneDefualtFileName.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0) ? string.Format("Dump of {0}, {1}", this.FileSystemName, DateTime.Now.ToString("yyyy-MM-dd HH-mm", CultureInfo.InvariantCulture)) : smugglerOptions.NoneDefualtFileName; result.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment") { FileName = fileName + ".ravendump" }; return(new CompletedTask <HttpResponseMessage>(result)); }
public async Task <HttpResponseMessage> ImportFilesystem(int batchSize, bool stripReplicationInformation, bool shouldDisableVersioningBundle) { if (!Request.Content.IsMimeMultipartContent()) { throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType); } string tempPath = FileSystem.Configuration.TempPath; var fullTempPath = tempPath + Constants.TempUploadsDirectoryName; if (File.Exists(fullTempPath)) { File.Delete(fullTempPath); } if (Directory.Exists(fullTempPath) == false) { Directory.CreateDirectory(fullTempPath); } var streamProvider = new MultipartFileStreamProvider(fullTempPath); await Request.Content.ReadAsMultipartAsync(streamProvider).ConfigureAwait(false); var uploadedFilePath = streamProvider.FileData[0].LocalFileName; string fileName = null; var fileContent = streamProvider.Contents.SingleOrDefault(); if (fileContent != null) { fileName = fileContent.Headers.ContentDisposition.FileName.Replace("\"", string.Empty); } var status = new DataDumperOperationStatus(); var cts = new CancellationTokenSource(); var task = Task.Run(async() => { try { var dataDumper = new FilesystemDataDumper(FileSystem); dataDumper.Progress += s => status.MarkProgress(s); var smugglerOptions = dataDumper.Options; smugglerOptions.BatchSize = batchSize; smugglerOptions.ShouldDisableVersioningBundle = shouldDisableVersioningBundle; smugglerOptions.StripReplicationInformation = stripReplicationInformation; smugglerOptions.CancelToken = cts; await dataDumper.ImportData(new SmugglerImportOptions <FilesConnectionStringOptions> { FromFile = uploadedFilePath }).ConfigureAwait(false); } catch (Exception e) { if (cts.Token.IsCancellationRequested) { status.MarkCanceled("Task was cancelled"); cts.Token.ThrowIfCancellationRequested(); //needed for displaying the task status as canceled and not faulted } if (e is InvalidDataException) { status.ExceptionDetails = e.Message; } else if (e is OperationVetoedException && e.Message.Contains(VersioningTriggerActions.CreationOfHistoricalRevisionIsNotAllowed)) { status.ExceptionDetails = "You are trying to import historical documents while the versioning bundle is enabled. " + "You should disable versioning during such import. Please mark the checkbox 'Disable versioning bundle during import' at Import File System"; } else { status.ExceptionDetails = e.ToString(); } status.MarkFaulted(status.ExceptionDetails); throw; } finally { status.MarkCompleted(); File.Delete(uploadedFilePath); } }, cts.Token); long id; FileSystem.Tasks.AddTask(task, status, new TaskActions.PendingTaskDescription { StartTime = SystemTime.UtcNow, TaskType = TaskActions.PendingTaskType.ImportFileSystem, Payload = fileName }, out id, cts); return(GetMessageWithObject(new { OperationId = id }, HttpStatusCode.Accepted)); }
public Task <HttpResponseMessage> ExportFilesystem(StudioTasksController.ExportData smugglerOptionsJson) { var result = GetEmptyMessage(); var taskId = smugglerOptionsJson.ProgressTaskId; var requestString = smugglerOptionsJson.DownloadOptions; SmugglerFilesOptions smugglerOptions; using (var jsonReader = new RavenJsonTextReader(new StringReader(requestString))) { var serializer = JsonExtensions.CreateDefaultJsonSerializer(); smugglerOptions = (SmugglerFilesOptions)serializer.Deserialize(jsonReader, typeof(SmugglerFilesOptions)); } var fileName = string.IsNullOrEmpty(smugglerOptions.NoneDefualtFileName) || (smugglerOptions.NoneDefualtFileName.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0) ? $"Dump of {FileSystemName}, {DateTime.Now.ToString("yyyy-MM-dd HH-mm", CultureInfo.InvariantCulture)}" : smugglerOptions.NoneDefualtFileName; //create PushStreamContent object that will be called when the output stream will be ready. result.Content = new PushStreamContent(async(outputStream, content, arg3) => { var status = new DataDumperOperationStatus(); var tcs = new TaskCompletionSource <object>(); var sp = Stopwatch.StartNew(); try { FileSystem.Tasks.AddTask(tcs.Task, status, new TaskActions.PendingTaskDescription { StartTime = SystemTime.UtcNow, TaskType = TaskActions.PendingTaskType.ExportFileSystem, Payload = "Exporting file system, file name: " + fileName }, taskId, smugglerOptions.CancelToken, skipStatusCheck: true); var dataDumper = new FilesystemDataDumper(FileSystem, smugglerOptions); dataDumper.Progress += s => status.MarkProgress(s); await dataDumper.ExportData( new SmugglerExportOptions <FilesConnectionStringOptions> { ToStream = outputStream }).ConfigureAwait(false); const string message = "Completed export"; status.MarkCompleted(message, sp.Elapsed); } catch (OperationCanceledException e) { status.MarkCanceled(e.Message); } catch (Exception e) { status.ExceptionDetails = e.ToString(); status.MarkFaulted(e.ToString()); throw; } finally { tcs.SetResult("Completed"); outputStream.Close(); } }); result.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment") { FileName = fileName + ".ravenfsdump" }; return(new CompletedTask <HttpResponseMessage>(result)); }
public async Task <HttpResponseMessage> ImportFilesystem(int batchSize) { if (!Request.Content.IsMimeMultipartContent()) { throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType); } string tempPath = Path.GetTempPath(); var fullTempPath = tempPath + Constants.TempUploadsDirectoryName; if (File.Exists(fullTempPath)) { File.Delete(fullTempPath); } if (Directory.Exists(fullTempPath) == false) { Directory.CreateDirectory(fullTempPath); } var streamProvider = new MultipartFileStreamProvider(fullTempPath); await Request.Content.ReadAsMultipartAsync(streamProvider).ConfigureAwait(false); var uploadedFilePath = streamProvider.FileData[0].LocalFileName; string fileName = null; var fileContent = streamProvider.Contents.SingleOrDefault(); if (fileContent != null) { fileName = fileContent.Headers.ContentDisposition.FileName.Replace("\"", string.Empty); } var status = new ImportOperationStatus(); var cts = new CancellationTokenSource(); var task = Task.Run(async() => { try { var dataDumper = new FilesystemDataDumper(FileSystem); dataDumper.Progress += s => status.LastProgress = s; var smugglerOptions = dataDumper.Options; smugglerOptions.BatchSize = batchSize; smugglerOptions.CancelToken = cts; await dataDumper.ImportData(new SmugglerImportOptions <FilesConnectionStringOptions> { FromFile = uploadedFilePath }); } catch (Exception e) { status.Faulted = true; status.State = RavenJObject.FromObject(new { Error = e.ToString() }); if (cts.Token.IsCancellationRequested) { status.State = RavenJObject.FromObject(new { Error = "Task was cancelled" }); cts.Token.ThrowIfCancellationRequested(); //needed for displaying the task status as canceled and not faulted } if (e is InvalidDataException) { status.ExceptionDetails = e.Message; } else { status.ExceptionDetails = e.ToString(); } throw; } finally { status.Completed = true; File.Delete(uploadedFilePath); } }, cts.Token); long id; FileSystem.Tasks.AddTask(task, status, new TaskActions.PendingTaskDescription { StartTime = SystemTime.UtcNow, TaskType = TaskActions.PendingTaskType.ImportFileSystem, Payload = fileName, }, out id, cts); return(GetMessageWithObject(new { OperationId = id })); }