public async Task export_result_cannot_contain_neither_files_scheduled_for_deletion_nor_tombstones(string storage)
{
var exportStream = new MemoryStream();
using (var store = this.NewStore(requestedStorage: storage))
{
using (var session = store.OpenAsyncSession())
{
for (int i = 0; i < 10; i++)
{
session.RegisterUpload("prefix-" + i + ".file", CreateUniformFileStream(10));
}
await session.SaveChangesAsync();
}
using (var session = store.OpenAsyncSession())
{
session.RegisterFileDeletion("prefix-3.file");
await session.SaveChangesAsync();
}
var fs = GetFileSystem();
var exporter = new FilesystemDataDumper(fs);
await exporter.ExportData(
new SmugglerExportOptions <FilesConnectionStringOptions>
{
ToStream = exportStream
}).ConfigureAwait(false);
using (var import = NewStore(1))
{
exportStream.Position = 0;
var importedFs = GetFileSystem(1);
var importer = new FilesystemDataDumper(importedFs);
await importer.ImportData(new SmugglerImportOptions <FilesConnectionStringOptions> {
FromStream = exportStream
});
importedFs.Storage.Batch(accessor =>
{
Assert.Equal(9, accessor.GetFilesAfter(Etag.Empty, 100).Count());
});
}
}
}
public Task <HttpResponseMessage> ExportFilesystem(StudioTasksController.ExportData smugglerOptionsJson)
{
var requestString = smugglerOptionsJson.SmugglerOptions;
SmugglerFilesOptions smugglerOptions;
using (var jsonReader = new RavenJsonTextReader(new StringReader(requestString)))
{
var serializer = JsonExtensions.CreateDefaultJsonSerializer();
smugglerOptions = (SmugglerFilesOptions)serializer.Deserialize(jsonReader, typeof(SmugglerFilesOptions));
}
var result = GetEmptyMessage();
// create PushStreamContent object that will be called when the output stream will be ready.
result.Content = new PushStreamContent(async(outputStream, content, arg3) =>
{
try
{
var dataDumper = new FilesystemDataDumper(FileSystem, smugglerOptions);
await dataDumper.ExportData(
new SmugglerExportOptions <FilesConnectionStringOptions>
{
ToStream = outputStream
}).ConfigureAwait(false);
}
finally
{
outputStream.Close();
}
});
var fileName = String.IsNullOrEmpty(smugglerOptions.NoneDefualtFileName) || (smugglerOptions.NoneDefualtFileName.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0) ?
string.Format("Dump of {0}, {1}", this.FileSystemName, DateTime.Now.ToString("yyyy-MM-dd HH-mm", CultureInfo.InvariantCulture)) :
smugglerOptions.NoneDefualtFileName;
result.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment")
{
FileName = fileName + ".ravendump"
};
return(new CompletedTask <HttpResponseMessage>(result));
}
public async Task <HttpResponseMessage> ImportFilesystem(int batchSize, bool stripReplicationInformation, bool shouldDisableVersioningBundle)
{
if (!Request.Content.IsMimeMultipartContent())
{
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
}
string tempPath = FileSystem.Configuration.TempPath;
var fullTempPath = tempPath + Constants.TempUploadsDirectoryName;
if (File.Exists(fullTempPath))
{
File.Delete(fullTempPath);
}
if (Directory.Exists(fullTempPath) == false)
{
Directory.CreateDirectory(fullTempPath);
}
var streamProvider = new MultipartFileStreamProvider(fullTempPath);
await Request.Content.ReadAsMultipartAsync(streamProvider).ConfigureAwait(false);
var uploadedFilePath = streamProvider.FileData[0].LocalFileName;
string fileName = null;
var fileContent = streamProvider.Contents.SingleOrDefault();
if (fileContent != null)
{
fileName = fileContent.Headers.ContentDisposition.FileName.Replace("\"", string.Empty);
}
var status = new DataDumperOperationStatus();
var cts = new CancellationTokenSource();
var task = Task.Run(async() =>
{
try
{
var dataDumper = new FilesystemDataDumper(FileSystem);
dataDumper.Progress += s => status.MarkProgress(s);
var smugglerOptions = dataDumper.Options;
smugglerOptions.BatchSize = batchSize;
smugglerOptions.ShouldDisableVersioningBundle = shouldDisableVersioningBundle;
smugglerOptions.StripReplicationInformation = stripReplicationInformation;
smugglerOptions.CancelToken = cts;
await dataDumper.ImportData(new SmugglerImportOptions <FilesConnectionStringOptions> {
FromFile = uploadedFilePath
}).ConfigureAwait(false);
}
catch (Exception e)
{
if (cts.Token.IsCancellationRequested)
{
status.MarkCanceled("Task was cancelled");
cts.Token.ThrowIfCancellationRequested(); //needed for displaying the task status as canceled and not faulted
}
if (e is InvalidDataException)
{
status.ExceptionDetails = e.Message;
}
else if (e is OperationVetoedException && e.Message.Contains(VersioningTriggerActions.CreationOfHistoricalRevisionIsNotAllowed))
{
status.ExceptionDetails = "You are trying to import historical documents while the versioning bundle is enabled. " +
"You should disable versioning during such import. Please mark the checkbox 'Disable versioning bundle during import' at Import File System";
}
else
{
status.ExceptionDetails = e.ToString();
}
status.MarkFaulted(status.ExceptionDetails);
throw;
}
finally
{
status.MarkCompleted();
File.Delete(uploadedFilePath);
}
}, cts.Token);
long id;
FileSystem.Tasks.AddTask(task, status, new TaskActions.PendingTaskDescription
{
StartTime = SystemTime.UtcNow,
TaskType = TaskActions.PendingTaskType.ImportFileSystem,
Payload = fileName
}, out id, cts);
return(GetMessageWithObject(new
{
OperationId = id
}, HttpStatusCode.Accepted));
}
public Task <HttpResponseMessage> ExportFilesystem(StudioTasksController.ExportData smugglerOptionsJson)
{
var result = GetEmptyMessage();
var taskId = smugglerOptionsJson.ProgressTaskId;
var requestString = smugglerOptionsJson.DownloadOptions;
SmugglerFilesOptions smugglerOptions;
using (var jsonReader = new RavenJsonTextReader(new StringReader(requestString)))
{
var serializer = JsonExtensions.CreateDefaultJsonSerializer();
smugglerOptions = (SmugglerFilesOptions)serializer.Deserialize(jsonReader, typeof(SmugglerFilesOptions));
}
var fileName = string.IsNullOrEmpty(smugglerOptions.NoneDefualtFileName) || (smugglerOptions.NoneDefualtFileName.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0) ?
$"Dump of {FileSystemName}, {DateTime.Now.ToString("yyyy-MM-dd HH-mm", CultureInfo.InvariantCulture)}" :
smugglerOptions.NoneDefualtFileName;
//create PushStreamContent object that will be called when the output stream will be ready.
result.Content = new PushStreamContent(async(outputStream, content, arg3) =>
{
var status = new DataDumperOperationStatus();
var tcs = new TaskCompletionSource <object>();
var sp = Stopwatch.StartNew();
try
{
FileSystem.Tasks.AddTask(tcs.Task, status, new TaskActions.PendingTaskDescription
{
StartTime = SystemTime.UtcNow,
TaskType = TaskActions.PendingTaskType.ExportFileSystem,
Payload = "Exporting file system, file name: " + fileName
}, taskId, smugglerOptions.CancelToken, skipStatusCheck: true);
var dataDumper = new FilesystemDataDumper(FileSystem, smugglerOptions);
dataDumper.Progress += s => status.MarkProgress(s);
await dataDumper.ExportData(
new SmugglerExportOptions <FilesConnectionStringOptions>
{
ToStream = outputStream
}).ConfigureAwait(false);
const string message = "Completed export";
status.MarkCompleted(message, sp.Elapsed);
}
catch (OperationCanceledException e)
{
status.MarkCanceled(e.Message);
}
catch (Exception e)
{
status.ExceptionDetails = e.ToString();
status.MarkFaulted(e.ToString());
throw;
}
finally
{
tcs.SetResult("Completed");
outputStream.Close();
}
});
result.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment")
{
FileName = fileName + ".ravenfsdump"
};
return(new CompletedTask <HttpResponseMessage>(result));
}
public async Task <HttpResponseMessage> ImportFilesystem(int batchSize)
{
if (!Request.Content.IsMimeMultipartContent())
{
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
}
string tempPath = Path.GetTempPath();
var fullTempPath = tempPath + Constants.TempUploadsDirectoryName;
if (File.Exists(fullTempPath))
{
File.Delete(fullTempPath);
}
if (Directory.Exists(fullTempPath) == false)
{
Directory.CreateDirectory(fullTempPath);
}
var streamProvider = new MultipartFileStreamProvider(fullTempPath);
await Request.Content.ReadAsMultipartAsync(streamProvider).ConfigureAwait(false);
var uploadedFilePath = streamProvider.FileData[0].LocalFileName;
string fileName = null;
var fileContent = streamProvider.Contents.SingleOrDefault();
if (fileContent != null)
{
fileName = fileContent.Headers.ContentDisposition.FileName.Replace("\"", string.Empty);
}
var status = new ImportOperationStatus();
var cts = new CancellationTokenSource();
var task = Task.Run(async() =>
{
try
{
var dataDumper = new FilesystemDataDumper(FileSystem);
dataDumper.Progress += s => status.LastProgress = s;
var smugglerOptions = dataDumper.Options;
smugglerOptions.BatchSize = batchSize;
smugglerOptions.CancelToken = cts;
await dataDumper.ImportData(new SmugglerImportOptions <FilesConnectionStringOptions> {
FromFile = uploadedFilePath
});
}
catch (Exception e)
{
status.Faulted = true;
status.State = RavenJObject.FromObject(new
{
Error = e.ToString()
});
if (cts.Token.IsCancellationRequested)
{
status.State = RavenJObject.FromObject(new { Error = "Task was cancelled" });
cts.Token.ThrowIfCancellationRequested(); //needed for displaying the task status as canceled and not faulted
}
if (e is InvalidDataException)
{
status.ExceptionDetails = e.Message;
}
else
{
status.ExceptionDetails = e.ToString();
}
throw;
}
finally
{
status.Completed = true;
File.Delete(uploadedFilePath);
}
}, cts.Token);
long id;
FileSystem.Tasks.AddTask(task, status, new TaskActions.PendingTaskDescription
{
StartTime = SystemTime.UtcNow,
TaskType = TaskActions.PendingTaskType.ImportFileSystem,
Payload = fileName,
}, out id, cts);
return(GetMessageWithObject(new
{
OperationId = id
}));
}
