public bool ImportDatabaseSmugglerApi(string databaseName, ItemType itemTypeToImport = ItemType.Documents)
{
var success = true;
try
{
if (string.IsNullOrWhiteSpace(databaseName))
{
_logger.Warning("Database name incorrectly");
success = false;
}
_logger.Information("Import database {0} with Smuggler Api", databaseName);
var filePath = GetFilePathFromDatabaseName(databaseName);
var filters = new List <FilterSetting>
{
new FilterSetting
{
Path = "@metadata.@id",
ShouldMatch = false,
Values = new List <string> {
"Raven/Encryption/Verification"
}
}
};
var smugglerApi = new SmugglerDatabaseApi(new SmugglerDatabaseOptions
{
OperateOnTypes = itemTypeToImport,
Incremental = false,
ShouldDisableVersioningBundle = true,
Filters = filters
});
var importOptions = new SmugglerImportOptions <RavenConnectionStringOptions>
{
FromFile = filePath,
To = new RavenConnectionStringOptions
{
DefaultDatabase = databaseName,
Url = _store.Url
},
};
smugglerApi.ImportData(importOptions).Wait();
}
catch (Exception ex)
{
_logger.Information("Import database failed: {0}", ex);
success = false;
}
return(success);
}
public async Task <HttpResponseMessage> Import()
{
// Make sure that we actually got the right data
if (!Request.Content.IsMimeMultipartContent())
{
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
}
try
{
// Store the uploaded file into a temporary location
var provider = new MultipartFormDataStreamProvider(Path.GetTempPath());
await Request.Content.ReadAsMultipartAsync(provider);
string filename = provider.FormData.GetValues("filename").First();
var file = provider.FileData.First();
// Setup an import using RavenDb's Smuggler API or the DatabaseDumper API depending on whether the embedded database is being used
SmugglerDatabaseApiBase importer;
RavenConnectionStringOptions connectionStringOptions;
if (Database.DocumentStore is EmbeddableDocumentStore embeddableDocumentStore)
{
importer = new DatabaseDataDumper(embeddableDocumentStore.DocumentDatabase);
connectionStringOptions = new EmbeddedRavenConnectionStringOptions();
}
else
{
importer = new SmugglerDatabaseApi();
connectionStringOptions = new RavenConnectionStringOptions()
{
Url = Database.DocumentStore.Url
};
}
var importOptions = new SmugglerImportOptions <RavenConnectionStringOptions>()
{
FromFile = file.LocalFileName,
To = connectionStringOptions
};
await importer.ImportData(importOptions);
return(new HttpResponseMessage(HttpStatusCode.OK));
}
catch (Exception exp)
{
return(this.Request.CreateErrorResponse(HttpStatusCode.InternalServerError, exp));
}
}
public override async Task ImportData(SmugglerImportOptions <FilesConnectionStringOptions> importOptions)
{
if (importOptions.To == null)
{
throw new ArgumentNullException("importOptions");
}
using (primaryStore = await CreateStore(importOptions.To))
using (documentStore = CreateDocumentStore(importOptions.To))
{
Operations = new SmugglerRemoteFilesOperations(() => primaryStore, () => documentStore);
await base.ImportData(importOptions);
}
}
public async Task <HttpResponseMessage> ImportDatabase(int batchSize, bool includeExpiredDocuments, ItemType operateOnTypes, string filtersPipeDelimited, string transformScript)
{
if (!this.Request.Content.IsMimeMultipartContent())
{
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
}
var streamProvider = new MultipartMemoryStreamProvider();
await Request.Content.ReadAsMultipartAsync(streamProvider);
var fileStream = await streamProvider.Contents
.First(c => c.Headers.ContentDisposition.Name == "\"file\"")
.ReadAsStreamAsync();
var dataDumper = new DataDumper(Database);
var importOptions = new SmugglerImportOptions
{
FromStream = fileStream
};
var options = new SmugglerOptions
{
BatchSize = batchSize,
ShouldExcludeExpired = includeExpiredDocuments,
OperateOnTypes = operateOnTypes,
TransformScript = transformScript
};
// Filters are passed in without the aid of the model binder. Instead, we pass in a list of FilterSettings using a string like this: pathHere;;;valueHere;;;true|||againPathHere;;;anotherValue;;;false
// Why? Because I don't see a way to pass a list of a values to a WebAPI method that accepts a file upload, outside of passing in a simple string value and parsing it ourselves.
if (filtersPipeDelimited != null)
{
options.Filters.AddRange(filtersPipeDelimited
.Split(new string[] { "|||" }, StringSplitOptions.RemoveEmptyEntries)
.Select(f => f.Split(new string[] { ";;;" }, StringSplitOptions.RemoveEmptyEntries))
.Select(o => new FilterSetting
{
Path = o[0],
Values = new List <string> {
o[1]
},
ShouldMatch = bool.Parse(o[2])
}));
}
await dataDumper.ImportData(importOptions, options);
return(GetEmptyMessage());
}
public async Task ExportShouldDisableSynchronizationDestinations()
{
using (var exportStream = new MemoryStream())
using (var exportStore = NewStore())
using (var importStore = NewStore(1))
{
await exportStore.AsyncFilesCommands.Synchronization.SetDestinationsAsync(new SynchronizationDestination()
{
ServerUrl = "http://sample.com",
FileSystem = "Sample",
Enabled = true
});
var exportOptions = new SmugglerExportOptions <FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = exportStore.Url,
DefaultFileSystem = exportStore.DefaultFileSystem
},
ToStream = exportStream
};
await new SmugglerFilesApi().ExportData(exportOptions);
exportStream.Position = 0;
var importOptions = new SmugglerImportOptions <FilesConnectionStringOptions>
{
FromStream = exportStream,
To = new FilesConnectionStringOptions()
{
Url = importStore.Url,
DefaultFileSystem = importStore.DefaultFileSystem
}
};
await new SmugglerFilesApi().ImportData(importOptions);
var destinations = await importStore.AsyncFilesCommands.Synchronization.GetDestinationsAsync();
Assert.Equal(1, destinations.Length);
Assert.Equal("http://sample.com/fs/Sample", destinations[0].Url);
Assert.Equal("Sample", destinations[0].FileSystem);
Assert.False(destinations[0].Enabled);
}
}
protected override void ImportDump()
{
// https://ravendb.net/docs/article-page/3.5/csharp/server/administration/exporting-and-importing-data#importing
var smugglerApi = new SmugglerDatabaseApi(new SmugglerDatabaseOptions
{
OperateOnTypes = ItemType.Documents,
Incremental = false
});
var importOptions = new SmugglerImportOptions <RavenConnectionStringOptions>
{
FromFile = o.Dump,
To = new RavenConnectionStringOptions
{
DefaultDatabase = o.Db,
Url = o.Url
}
};
smugglerApi.ImportData(importOptions).GetAwaiter().GetResult();
GetDbSize();
}
public override async Task ImportData(SmugglerImportOptions <RavenConnectionStringOptions> importOptions, Stream stream)
{
using (store = CreateStore(importOptions.To))
{
Task disposeTask;
try
{
await CreateBulkInsertOperation();
await base.ImportData(importOptions, stream);
}
finally
{
disposeTask = operation.DisposeAsync();
}
if (disposeTask != null)
{
await disposeTask;
}
}
}
/// <summary>
/// Import counter data from a dump file
/// </summary>
/// <param name="importOptions">options that specify the source and destination of the data</param>
/// <exception cref="ArgumentException">FromXXXX, To, Url and CounterStoreId parameters must be present in the import options</exception>
public async Task ImportData(SmugglerImportOptions <CounterConnectionStringOptions> importOptions)
{
if (string.IsNullOrWhiteSpace(importOptions.FromFile) && importOptions.FromStream == null)
{
throw new ArgumentException("Missing from parameter from import options - be sure to define either FromFile or FromStream property");
}
if (importOptions.To == null)
{
throw new ArgumentException("Missing To parameter from importOptions - do not know where to import to.");
}
if (string.IsNullOrWhiteSpace(importOptions.To.Url))
{
throw new ArgumentException("Missing Url of the RavenDB server - do not know where to import to");
}
if (string.IsNullOrWhiteSpace(importOptions.To.CounterStoreId))
{
throw new ArgumentException("Missing Id of the Counter Store - do not know where to import to");
}
if (Options.Incremental == false)
{
var stream = importOptions.FromStream;
var ownStream = false;
try
{
if (stream == null)
{
stream = File.OpenRead(importOptions.FromFile);
ShowProgress($"Starting full import from file : {importOptions.FromFile}");
ownStream = true;
}
else
{
ShowProgress("Starting full import from stream");
}
await ImportFullData(importOptions.To, stream).WithCancellation(CancellationToken).ConfigureAwait(false);
}
finally
{
if (stream != null && ownStream)
{
stream.Dispose();
}
}
}
else
{
var dumpFilePath = Path.GetFullPath(importOptions.FromFile);
ShowProgress("Enumerating incremental dump files at " + dumpFilePath);
var files = Directory.GetFiles(dumpFilePath)
.Where(file => CounterIncrementalDump.Equals(Path.GetExtension(file), StringComparison.CurrentCultureIgnoreCase))
.OrderBy(File.GetLastWriteTimeUtc)
.ToArray();
if (files.Length == 0)
{
return;
}
foreach (var file in files)
{
using (var fileStream = File.OpenRead(Path.Combine(importOptions.FromFile, file)))
{
ShowProgress($"Starting incremental import from file: {file}");
await ImportIncrementalData(importOptions.To, fileStream).WithCancellation(CancellationToken).ConfigureAwait(false);
}
}
}
}
public async Task ShouldExportAndImportConfigurations()
{
using (var exportStream = new MemoryStream())
{
int countOfConfigurations;
using (var store = NewStore())
{
for (int i = 0; i < 100; i++)
{
await store.AsyncFilesCommands.Configuration.SetKeyAsync("items/" + i, new RavenJObject
{
{
"test", "value"
},
{
"test-array", new RavenJArray
{
"item-1", "item-2", "item-3"
}
}
});
}
countOfConfigurations = (await store.AsyncFilesCommands.Configuration.GetKeyNamesAsync(0, 200)).Length;
var exportOptions = new SmugglerExportOptions <FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = store.DefaultFileSystem
},
ToStream = exportStream
};
await new SmugglerFilesApi().ExportData(exportOptions);
}
using (var import = NewStore(1))
{
exportStream.Position = 0;
var importOptions = new SmugglerImportOptions <FilesConnectionStringOptions>
{
FromStream = exportStream,
To = new FilesConnectionStringOptions()
{
Url = import.Url,
DefaultFileSystem = import.DefaultFileSystem
}
};
await new SmugglerFilesApi().ImportData(importOptions);
Assert.Equal(countOfConfigurations, (await import.AsyncFilesCommands.Configuration.GetKeyNamesAsync(0, 200)).Length);
for (int i = 0; i < 100; i++)
{
Assert.NotNull(await import.AsyncFilesCommands.Configuration.GetKeyAsync <RavenJObject>("items/" + i));
}
}
}
}
public async Task Foo()
{
{
#region smuggler_api_1
SmugglerFilesApi smugglerApi = new SmugglerFilesApi();
var exportOptions = new SmugglerExportOptions <FilesConnectionStringOptions>
{
ToFile = "dump.ravenfs",
From = new FilesConnectionStringOptions
{
Url = "http://localhost:8080",
DefaultFileSystem = "NorthwindFS"
}
};
var exportResult = await smugglerApi.ExportData(exportOptions);
#endregion
}
{
#region smuggler_api_2
SmugglerFilesApi smugglerApi = new SmugglerFilesApi();
var importOptions = new SmugglerImportOptions <FilesConnectionStringOptions>
{
FromFile = "dump.ravenfs",
To = new FilesConnectionStringOptions
{
Url = "http://localhost:8080",
DefaultFileSystem = "NewNorthwindFS"
}
};
await smugglerApi.ImportData(importOptions);
#endregion
}
{
#region smuggler_api_3
// export files
// from NorthwindFS file system
// found on http://localhost:8080
// and import them to NewNorthwindFS
// found on the same server
SmugglerFilesApi smugglerApi = new SmugglerFilesApi();
var betweenOptions = new SmugglerBetweenOptions <FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = "http://localhost:8080",
DefaultFileSystem = "NorthwindFS"
},
To = new FilesConnectionStringOptions
{
Url = "http://localhost:8080",
DefaultFileSystem = "NewNorthwindFS"
}
};
await smugglerApi.Between(betweenOptions);
#endregion
}
}
public async Task Foo()
{
{
#region smuggler_api_1
SmugglerFilesApi smugglerApi = new SmugglerFilesApi();
var exportOptions = new SmugglerExportOptions<FilesConnectionStringOptions>
{
ToFile = "dump.ravenfs",
From = new FilesConnectionStringOptions
{
Url = "http://localhost:8080",
DefaultFileSystem = "NorthwindFS"
}
};
var exportResult = await smugglerApi.ExportData(exportOptions);
#endregion
}
{
#region smuggler_api_2
SmugglerFilesApi smugglerApi = new SmugglerFilesApi();
var importOptions = new SmugglerImportOptions<FilesConnectionStringOptions>
{
FromFile = "dump.ravenfs",
To = new FilesConnectionStringOptions
{
Url = "http://localhost:8080",
DefaultFileSystem = "NewNorthwindFS"
}
};
await smugglerApi.ImportData(importOptions);
#endregion
}
{
#region smuggler_api_3
// export files
// from NorthwindFS file system
// found on http://localhost:8080
// and import them to NewNorthwindFS
// found on the same server
SmugglerFilesApi smugglerApi = new SmugglerFilesApi();
var betweenOptions = new SmugglerBetweenOptions<FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = "http://localhost:8080",
DefaultFileSystem = "NorthwindFS"
},
To = new FilesConnectionStringOptions
{
Url = "http://localhost:8080",
DefaultFileSystem = "NewNorthwindFS"
}
};
await smugglerApi.Between(betweenOptions);
#endregion
}
}
