public async Task GetImport()
{
if (HttpContext.Request.Query.ContainsKey("file") == false &&
HttpContext.Request.Query.ContainsKey("url") == false)
{
throw new ArgumentException("'file' or 'url' are mandatory when using GET /smuggler/import");
}
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var options = DatabaseSmugglerOptionsServerSide.Create(HttpContext);
using (var stream = new GZipStream(new BufferedStream(await GetImportStream(), 128 * Voron.Global.Constants.Size.Kilobyte), CompressionMode.Decompress))
using (var token = CreateOperationToken())
using (var source = new StreamSource(stream, context, Database))
{
var destination = new DatabaseDestination(Database);
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, token: token.Token);
var result = smuggler.Execute();
WriteImportResult(context, result, ResponseBodyStream());
}
}
}
public void Northwind_RemoteToFile_FileToRemote_Test()
{
using (var store1 = CreateStore())
using (var store2 = CreateStore())
{
DeployNorthwind(store1);
WaitForIndexing(store1);
var outputFile = Path.Combine(NewDataPath(forceCreateDir: true), "backup.ravendump");
var smuggler = new DatabaseSmuggler(new DatabaseSmugglerOptions(), new DatabaseSmugglerRemoteSource(store1), new DatabaseSmugglerFileDestination(outputFile));
smuggler.Execute();
smuggler = new DatabaseSmuggler(new DatabaseSmugglerOptions(), new DatabaseSmugglerFileSource(outputFile), new DatabaseSmugglerRemoteDestination(store2));
smuggler.Execute();
WaitForIndexing(store2);
var statistics = store2.DatabaseCommands.GetStatistics();
Assert.Equal(1059, statistics.CountOfDocuments);
Assert.Equal(4, statistics.CountOfIndexes);
Assert.Equal(1, statistics.CountOfResultTransformers);
}
}
private static void DeployNorthwindAndExportToFile(DocumentStore store, string path)
{
DeployNorthwind(store);
var smuggler = new DatabaseSmuggler(new DatabaseSmugglerOptions(), new DatabaseSmugglerRemoteSource(store), new DatabaseSmugglerFileDestination(path));
smuggler.Execute();
}
private IOperationResult ExportDatabaseInternal(DatabaseSmugglerOptions options, Action <IOperationProgress> onProgress, DocumentsOperationContext context, OperationCancelToken token)
{
using (token)
{
var source = new DatabaseSource(Database, 0);
var destination = new StreamDestination(ResponseBodyStream(), context, source);
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, onProgress: onProgress, token: token.Token);
return(smuggler.Execute());
}
}
private void DoImportInternal(DocumentsOperationContext context, Stream stream, DatabaseSmugglerOptionsServerSide options, SmugglerResult result, Action<IOperationProgress> onProgress, OperationCancelToken token)
{
using (stream)
using (token)
using (var source = new StreamSource(stream, context, Database))
{
var destination = new DatabaseDestination(Database);
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, result, onProgress, token.Token);
smuggler.Execute();
}
}
private void ImportDocumentsFromCsvStream(Stream stream, DocumentsOperationContext context, string entity, DatabaseSmugglerOptionsServerSide options, SmugglerResult result, Action<IOperationProgress> onProgress, OperationCancelToken token)
{
if (string.IsNullOrEmpty(entity) == false && char.IsLower(entity[0]))
entity = char.ToUpper(entity[0]) + entity.Substring(1);
result.AddInfo($"Import collection: {entity}");
using (var source = new CsvStreamSource(Database, stream, context, entity))
{
var destination = new DatabaseDestination(Database);
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, result, onProgress, token.Token);
smuggler.Execute();
}
}
public void ShouldSmuggleOnlyTransformers()
{
var destination = new DatabaseSmugglerCountingDestination();
var smuggler = new DatabaseSmuggler(new DatabaseSmugglerOptions
{
OperateOnTypes = DatabaseItemType.Transformers
}, new DatabaseSmugglerFileSource(_path), destination);
smuggler.Execute();
Assert.Equal(0, destination.WroteDocuments);
Assert.Equal(0, destination.WroteDocumentDeletions);
Assert.Equal(0, destination.WroteIdentities);
Assert.Equal(0, destination.WroteIndexes);
Assert.Equal(1, destination.WroteTransformers);
}
private IOperationResult ExportDatabaseInternal(
DatabaseSmugglerOptionsServerSide options,
long startDocumentEtag,
long startRaftIndex,
Action <IOperationProgress> onProgress,
DocumentsOperationContext context,
OperationCancelToken token)
{
using (token)
{
var source = new DatabaseSource(Database, startDocumentEtag, startRaftIndex);
using (var outputStream = GetOutputStream(ResponseBodyStream(), options))
{
var destination = new StreamDestination(outputStream, context, source);
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, onProgress: onProgress, token: token.Token);
return(smuggler.Execute());
}
}
}
public async Task PostImport()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var options = DatabaseSmugglerOptionsServerSide.Create(HttpContext, context);
using (var stream = new GZipStream(new BufferedStream(await GetImportStream(), 128 * Voron.Global.Constants.Size.Kilobyte), CompressionMode.Decompress))
using (var token = CreateOperationToken())
using (var source = new StreamSource(stream, context))
{
var destination = new DatabaseDestination(Database);
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, token: token.Token);
var result = smuggler.Execute();
WriteImportResult(context, result, ResponseBodyStream());
}
}
}
public void ShouldSmuggleOnlyTransformers()
{
using (var store = NewRemoteDocumentStore())
{
DeployNorthwind(store);
var destination = new DatabaseSmugglerCountingDestination();
var smuggler = new DatabaseSmuggler(new DatabaseSmugglerOptions
{
OperateOnTypes = DatabaseItemType.Transformers
}, new DatabaseSmugglerRemoteSource(store), destination);
smuggler.Execute();
Assert.Equal(0, destination.WroteDocuments);
Assert.Equal(0, destination.WroteDocumentDeletions);
Assert.Equal(0, destination.WroteIdentities);
Assert.Equal(0, destination.WroteIndexes);
Assert.Equal(1, destination.WroteTransformers);
}
}
public void ShouldSmuggleOnlyTransformers()
{
using (var store = NewRemoteDocumentStore())
{
var database = servers[0].Options.DatabaseLandlord.GetResourceInternal(store.DefaultDatabase).Result;
DeployNorthwind(store);
var destination = new DatabaseSmugglerCountingDestination();
var smuggler = new DatabaseSmuggler(new DatabaseSmugglerOptions
{
OperateOnTypes = DatabaseItemType.Transformers
}, new DatabaseSmugglerEmbeddedSource(database), destination);
smuggler.Execute();
Assert.Equal(0, destination.WroteDocuments);
Assert.Equal(0, destination.WroteDocumentDeletions);
Assert.Equal(0, destination.WroteIdentities);
Assert.Equal(0, destination.WroteIndexes);
Assert.Equal(1, destination.WroteTransformers);
}
}
private async Task BulkImport(BlockingCollection <Func <Task <Stream> > > files, string directory)
{
var results = new ConcurrentQueue <SmugglerResult>();
var tasks = new Task[Math.Max(1, ProcessorInfo.ProcessorCount / 2)];
var finalResult = new SmugglerResult();
for (int i = 0; i < tasks.Length; i++)
{
tasks[i] = Task.Run(async() =>
{
while (files.IsCompleted == false)
{
Func <Task <Stream> > getFile;
try
{
getFile = files.Take();
}
catch (Exception)
{
//TODO : add logging, _silently_ skipping is a bad idea
continue;
}
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var file = await getFile())
using (var stream = new GZipStream(new BufferedStream(file, 128 * Voron.Global.Constants.Size.Kilobyte), CompressionMode.Decompress))
using (var source = new StreamSource(stream, context))
{
var destination = new DatabaseDestination(Database);
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time);
var result = smuggler.Execute();
results.Enqueue(result);
}
}
});
}
await Task.WhenAll(tasks);
while (results.TryDequeue(out SmugglerResult importResult))
{
finalResult.Documents.SkippedCount += importResult.Documents.SkippedCount;
finalResult.Documents.ReadCount += importResult.Documents.ReadCount;
finalResult.Documents.ErroredCount += importResult.Documents.ErroredCount;
finalResult.Documents.LastEtag = Math.Max(finalResult.Documents.LastEtag, importResult.Documents.LastEtag);
finalResult.Documents.Attachments = importResult.Documents.Attachments;
finalResult.RevisionDocuments.ReadCount += importResult.RevisionDocuments.ReadCount;
finalResult.RevisionDocuments.ErroredCount += importResult.RevisionDocuments.ErroredCount;
finalResult.RevisionDocuments.LastEtag = Math.Max(finalResult.RevisionDocuments.LastEtag, importResult.RevisionDocuments.LastEtag);
finalResult.RevisionDocuments.Attachments = importResult.RevisionDocuments.Attachments;
finalResult.Identities.ReadCount += importResult.Identities.ReadCount;
finalResult.Identities.ErroredCount += importResult.Identities.ErroredCount;
finalResult.Indexes.ReadCount += importResult.Indexes.ReadCount;
finalResult.Indexes.ErroredCount += importResult.Indexes.ErroredCount;
foreach (var message in importResult.Messages)
{
finalResult.AddMessage(message);
}
}
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext finalContext))
{
var memoryStream = new MemoryStream();
WriteImportResult(finalContext, finalResult, memoryStream);
memoryStream.Position = 0;
try
{
using (var output = File.Create(Path.Combine(directory, "smuggler.results.txt")))
{
memoryStream.CopyTo(output);
}
}
catch (Exception)
{
// ignore any failure here
}
memoryStream.Position = 0;
memoryStream.CopyTo(ResponseBodyStream());
}
}
