public async Task ShouldNotThrowIfFileSystemExists()
{
using (var store = NewStore(fileSystemName: "DoesExists", requestedStorage: "esent"))
{
var server = GetServer();
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
try
{
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(store.DefaultFileSystem);
await InitializeWithRandomFiles(store, 2, 4);
var options = new FilesConnectionStringOptions {
Url = store.Url, DefaultFileSystem = store.DefaultFileSystem
};
var smugglerApi = new SmugglerFilesApi();
var export = await smugglerApi.ExportData(new SmugglerExportOptions <FilesConnectionStringOptions> {
From = options, ToFile = outputDirectory
});
await smugglerApi.ImportData(new SmugglerImportOptions <FilesConnectionStringOptions> {
FromFile = export.FilePath, To = options
});
}
finally
{
IOExtensions.DeleteDirectory(outputDirectory);
}
}
}
public async Task BetweenOperation_ShouldNotCreateFilesystem()
{
using (var store = NewStore())
{
var smugglerApi = new SmugglerFilesApi();
var options = new SmugglerBetweenOptions <FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = SourceFilesystem
},
To = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = DestinationFilesystem
}
};
var exception = await AssertAsync.Throws <SmugglerException>(() => smugglerApi.Between(options));
Assert.True(exception.Message.StartsWith("Smuggler does not support file system creation (file system '" + SourceFilesystem + "' on server"));
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(SourceFilesystem);
exception = await AssertAsync.Throws <SmugglerException>(() => smugglerApi.Between(options));
Assert.True(exception.Message.StartsWith("Smuggler does not support file system creation (file system '" + DestinationFilesystem + "' on server"));
}
}
private async Task VerifyDump(FilesStore store, string backupPath, Action <FilesStore> action)
{
try
{
store.DefaultFileSystem += "-Verify";
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(store.DefaultFileSystem);
bool incremental = false;
var directory = new DirectoryInfo(backupPath);
if (directory.Exists)
{
incremental = true;
}
var dumper = new SmugglerFilesApi {
Options = { Incremental = incremental }
};
await dumper.ImportData(new SmugglerImportOptions <FilesConnectionStringOptions>
{
FromFile = backupPath,
To = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = store.DefaultFileSystem,
}
});
action(store);
}
catch (Exception e)
{
throw e.SimplifyException();
}
}
public async Task BehaviorWhenServerIsDown()
{
using (var store = NewStore())
{
var server = GetServer();
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
try
{
var options = new FilesConnectionStringOptions {
Url = "http://localhost:8078/", DefaultFileSystem = store.DefaultFileSystem
};
var smugglerApi = new SmugglerFilesApi();
var e = await AssertAsync.Throws <SmugglerException>(() => smugglerApi.ImportData(new SmugglerImportOptions <FilesConnectionStringOptions> {
FromFile = outputDirectory.ToFullPath(), To = options
}));
Assert.Contains("Smuggler encountered a connection problem:", e.Message);
e = await AssertAsync.Throws <SmugglerException>(() => smugglerApi.ExportData(new SmugglerExportOptions <FilesConnectionStringOptions> {
ToFile = outputDirectory.ToFullPath(), From = options
}));
Assert.Contains("Smuggler encountered a connection problem:", e.Message);
}
finally
{
IOExtensions.DeleteDirectory(outputDirectory);
}
}
}
public async Task BetweenOperation_MetadataIsPreserved()
{
using (var store = NewStore())
{
var server = GetServer();
int fileSize = 10000;
var smugglerApi = new SmugglerFilesApi();
var options = new SmugglerBetweenOptions <FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = SourceFilesystem
},
To = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = DestinationFilesystem
}
};
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(SourceFilesystem);
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(DestinationFilesystem);
FileHeader originalFile;
using (var session = store.OpenAsyncSession(SourceFilesystem))
{
session.RegisterUpload("test1.file", CreateRandomFileStream(12800));
await session.SaveChangesAsync();
// content update after a metadata change
originalFile = await session.LoadFileAsync("test1.file");
originalFile.Metadata["Test"] = new RavenJValue("Value");
await session.SaveChangesAsync();
}
await smugglerApi.Between(options);
using (var session = store.OpenAsyncSession(DestinationFilesystem))
{
var file = await session.LoadFileAsync("test1.file");
Assert.Equal(originalFile.CreationDate, file.CreationDate);
Assert.Equal(originalFile.Directory, file.Directory);
Assert.Equal(originalFile.Extension, file.Extension);
Assert.Equal(originalFile.FullPath, file.FullPath);
Assert.Equal(originalFile.Name, file.Name);
Assert.Equal(originalFile.TotalSize, file.TotalSize);
Assert.Equal(originalFile.UploadedSize, file.UploadedSize);
Assert.Equal(originalFile.LastModified, file.LastModified);
Assert.True(file.Metadata.ContainsKey("Test"));
}
}
}
public async Task BetweenOperation_ShouldNotCreateFilesystem()
{
string export = Path.Combine(NewDataPath("export_5584"), "Export");
using (var store = NewStore())
{
var ms = new MemoryStream();
var buffer = new byte[1024];
for (int i = 0; i < buffer.Length; i++)
{
buffer[i] = (byte)i;
}
ms.Write(buffer, 0, buffer.Length);
ms.Position = 0;
await store.AsyncFilesCommands.UploadAsync("file.txt", ms);
var tasks = new Task[50];
for (var i = 0; i < 100; i += tasks.Length)
{
for (var j = 0; j < tasks.Length; j++)
{
var targetNAme = "file" + (i + j).ToString() + ".txt";
tasks[j] = store.AsyncFilesCommands.CopyAsync("file.txt", targetNAme);
}
Task.WaitAll(tasks);
}
var smugglerApi = new SmugglerFilesApi();
await smugglerApi.ExportData(exportOptions : new SmugglerExportOptions <FilesConnectionStringOptions>()
{
From = new FilesConnectionStringOptions()
{
DefaultFileSystem = store.DefaultFileSystem,
Url = store.Url,
},
ToFile = export,
});
using (var store2 = NewStore(fileSystemName: "export", index: 1))
{
var importSmugglerApi = new SmugglerFilesApi();
await importSmugglerApi.ImportData(importOptions : new SmugglerImportOptions <FilesConnectionStringOptions>()
{
To = new FilesConnectionStringOptions()
{
DefaultFileSystem = store2.DefaultFileSystem,
Url = store2.Url,
},
FromFile = export,
});
}
}
}
public async Task BetweenOperation_ContentIsPreserved_MultipleDirectories()
{
using (var store = NewStore())
{
var server = GetServer();
int fileSize = 10000;
var smugglerApi = new SmugglerFilesApi();
var options = new SmugglerBetweenOptions <FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = SourceFilesystem
},
To = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = DestinationFilesystem
}
};
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(SourceFilesystem);
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(DestinationFilesystem);
var files = new Stream[10];
using (var session = store.OpenAsyncSession(SourceFilesystem))
{
for (int i = 0; i < files.Length; i++)
{
files[i] = CreateRandomFileStream(100 * i + 12);
session.RegisterUpload(i + "/test.file", files[i]);
}
await session.SaveChangesAsync();
}
await smugglerApi.Between(options);
for (int i = 0; i < files.Length; i++)
{
using (var session = store.OpenAsyncSession(DestinationFilesystem))
{
var file = await session.LoadFileAsync(i + "/test.file");
var stream = await session.DownloadAsync(file);
files[i].Position = 0;
Assert.Equal(files[i].GetMD5Hash(), stream.GetMD5Hash());
}
}
}
}
public async Task CanDisableVersioningDuringImport_Between()
{
using (var storeExport = NewStore())
using (var storeImport = NewStore(activeBundles: "Versioning", fileSystemName: "Import", index: 1))
{
for (int i = 0; i < 10; i++)
{
await storeExport.AsyncFilesCommands.UploadAsync("test-" + i, StringToStream("hello"));
}
await storeImport.AsyncFilesCommands.Configuration.SetKeyAsync(VersioningUtil.DefaultConfigurationName, new VersioningConfiguration
{
Id = VersioningUtil.DefaultConfigurationName
});
var fileCount = (await storeExport.AsyncFilesCommands.GetStatisticsAsync()).FileCount;
var smuggler = new SmugglerFilesApi(new SmugglerFilesOptions()
{
ShouldDisableVersioningBundle = true
});
var options = new SmugglerBetweenOptions <FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = storeExport.Url,
DefaultFileSystem = storeExport.DefaultFileSystem
},
To = new FilesConnectionStringOptions
{
Url = storeImport.Url,
DefaultFileSystem = storeImport.DefaultFileSystem
}
};
await smuggler.Between(options);
var fileCountAfterImport = (await storeImport.AsyncFilesCommands.GetStatisticsAsync()).FileCount;
Assert.Equal(fileCount, fileCountAfterImport);
// after import versioning should be active
await storeImport.AsyncFilesCommands.UploadAsync("with-rev", StringToStream("rev"));
using (var session = storeImport.OpenAsyncSession())
{
var revisions = await session.GetRevisionsForAsync("with-rev", 0, 10);
Assert.Equal(1, revisions.Length);
}
}
}
public async Task BetweenOperation_BehaviorWhenServerIsDown()
{
string dataDirectory = null;
try
{
using (var store = NewStore())
{
var server = GetServer();
var smugglerApi = new SmugglerFilesApi();
var options = new SmugglerBetweenOptions <FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = "http://localhost:8078/",
DefaultFileSystem = SourceFilesystem
},
To = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = DestinationFilesystem
}
};
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(SourceFilesystem);
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(DestinationFilesystem);
var e = await AssertAsync.Throws <SmugglerException>(() => smugglerApi.Between(options));
Assert.Contains("Smuggler encountered a connection problem:", e.Message);
options.From.Url = store.Url;
options.To.Url = "http://localhost:8078/";
e = await AssertAsync.Throws <SmugglerException>(() => smugglerApi.Between(options));
Assert.Contains("Smuggler encountered a connection problem:", e.Message);
server.Dispose();
}
}
finally
{
if (!string.IsNullOrWhiteSpace(dataDirectory))
{
IOExtensions.DeleteDirectory(dataDirectory);
}
}
}
public async Task disableVersioningDuringImportFs()
{
var export = Path.Combine(NewDataPath("src"), "Export");
using (var store = NewStore(activeBundles: "Versioning"))
{
await store.AsyncFilesCommands.Configuration.SetKeyAsync(VersioningUtil.DefaultConfigurationName, new VersioningConfiguration
{
Id = VersioningUtil.DefaultConfigurationName
});
for (int i = 0; i < 10; i++)
{
await store.AsyncFilesCommands.UploadAsync("test-" + i, StringToStream("hello"));
}
var options = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = store.DefaultFileSystem
};
var smuggler = new SmugglerFilesApi();
await smuggler.ExportData(new SmugglerExportOptions <FilesConnectionStringOptions>
{
From = options,
ToFile = export
});
}
using (var store = NewStore(activeBundles: "Versioning", fileSystemName: "Import", index: 1))
{
await store.AsyncFilesCommands.Configuration.SetKeyAsync(VersioningUtil.DefaultConfigurationName, new VersioningConfiguration
{
Id = VersioningUtil.DefaultConfigurationName
});
var smuggler = new SmugglerFilesApi(new SmugglerFilesOptions()
{
ShouldDisableVersioningBundle = true
});
var options = new FilesConnectionStringOptions {
Url = store.Url, DefaultFileSystem = store.DefaultFileSystem
};
var e = await AssertAsync.DoesNotThrow(async() => await smuggler.ImportData(new SmugglerImportOptions <FilesConnectionStringOptions> {
FromFile = export, To = options
}));
Assert.True(e);
}
}
public async Task enableVersioningDuringImportFs()
{
var export = Path.Combine(NewDataPath("src"), "Export");
using (var store = NewStore(activeBundles: "Versioning"))
{
await store.AsyncFilesCommands.Configuration.SetKeyAsync(VersioningUtil.DefaultConfigurationName, new VersioningConfiguration
{
Id = VersioningUtil.DefaultConfigurationName
});
for (int i = 0; i < 10; i++)
{
await store.AsyncFilesCommands.UploadAsync("test-" + i, StringToStream("hello"));
}
var options = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = store.DefaultFileSystem
};
var smuggler = new SmugglerFilesApi();
await smuggler.ExportData(new SmugglerExportOptions <FilesConnectionStringOptions>
{
From = options,
ToFile = export
});
}
using (var store = NewStore(activeBundles: "Versioning", fileSystemName: "Import", index: 1))
{
await store.AsyncFilesCommands.Configuration.SetKeyAsync(VersioningUtil.DefaultConfigurationName, new VersioningConfiguration
{
Id = VersioningUtil.DefaultConfigurationName
});
var smuggler = new SmugglerFilesApi(new SmugglerFilesOptions()
{
ShouldDisableVersioningBundle = false
});
var options = new FilesConnectionStringOptions {
Url = store.Url, DefaultFileSystem = store.DefaultFileSystem
};
var e = await AssertAsync.Throws <ErrorResponseException>(async() => await smuggler.ImportData(new SmugglerImportOptions <FilesConnectionStringOptions> {
FromFile = export, To = options
}));
Assert.Contains("PUT vetoed on file /test-0/revisions/1 by Raven.Database.FileSystem.Bundles.Versioning.Plugins.VersioningPutTrigger because: Creating a historical revision is not allowed", e.Message);
}
}
public async Task BetweenOperation_ContentIsPreserved_SingleFile()
{
using (var store = NewStore())
{
var server = GetServer();
ReseedRandom(100); // Force a random distribution.
int fileSize = 10000;
var smugglerApi = new SmugglerFilesApi();
var options = new SmugglerBetweenOptions <FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = SourceFilesystem
},
To = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = DestinationFilesystem
}
};
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(SourceFilesystem);
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(DestinationFilesystem);
var fileContent = CreateRandomFileStream(fileSize);
using (var session = store.OpenAsyncSession(SourceFilesystem))
{
session.RegisterUpload("test1.file", fileContent);
await session.SaveChangesAsync();
}
await smugglerApi.Between(options);
fileContent.Position = 0;
using (var session = store.OpenAsyncSession(DestinationFilesystem))
{
var file = session.LoadFileAsync("test1.file").Result;
Assert.Equal(fileSize, file.TotalSize);
var stream = session.DownloadAsync(file).Result;
Assert.Equal(fileContent.GetMD5Hash(), stream.GetMD5Hash());
}
}
}
public async Task BetweenOperation_CanDumpEmptyFileSystem()
{
string dataDirectory = null;
try
{
using (var store = NewStore())
{
var server = GetServer();
dataDirectory = server.Configuration.DataDirectory;
var smugglerApi = new SmugglerFilesApi();
var options = new SmugglerBetweenOptions <FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = SourceFilesystem
},
To = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = DestinationFilesystem
}
};
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(SourceFilesystem);
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(DestinationFilesystem);
await smugglerApi.Between(options);
using (var session = store.OpenAsyncSession(DestinationFilesystem))
{
var files = await session.Commands.BrowseAsync();
Assert.Equal(0, files.Count());
}
server.Dispose();
}
}
finally
{
if (!string.IsNullOrWhiteSpace(dataDirectory))
{
IOExtensions.DeleteDirectory(dataDirectory);
}
}
}
public async Task ShouldSmuggleConfigurationsInBetweenOperation()
{
using (var exportStore = NewStore())
using (var importStore = NewStore(1))
{
for (int i = 0; i < 100; i++)
{
await exportStore.AsyncFilesCommands.Configuration.SetKeyAsync("items/" + i, new RavenJObject
{
{
"test", "value"
},
{
"test-array", new RavenJArray
{
"item-1", "item-2", "item-3"
}
}
});
}
var countOfConfigurations = (await exportStore.AsyncFilesCommands.Configuration.GetKeyNamesAsync(0, 200)).Length;
var options = new SmugglerBetweenOptions <FilesConnectionStringOptions>()
{
From = new FilesConnectionStringOptions()
{
Url = exportStore.Url,
DefaultFileSystem = exportStore.DefaultFileSystem
},
To = new FilesConnectionStringOptions()
{
Url = importStore.Url,
DefaultFileSystem = importStore.DefaultFileSystem
}
};
var smugglerFilesApi = new SmugglerFilesApi();
smugglerFilesApi.Options.BatchSize = 5;
await smugglerFilesApi.Between(options);
Assert.Equal(countOfConfigurations, (await importStore.AsyncFilesCommands.Configuration.GetKeyNamesAsync(0, 200)).Length);
for (int i = 0; i < 100; i++)
{
Assert.NotNull(await importStore.AsyncFilesCommands.Configuration.GetKeyAsync <RavenJObject>("items/" + i));
}
}
}
public async Task ShouldNotThrowIfFileSystemExistsUsingDefaultConfiguration()
{
using (var store = NewStore())
{
var server = GetServer();
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
try
{
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(store.DefaultFileSystem);
await InitializeWithRandomFiles(store, 1, 100);
var options = new FilesConnectionStringOptions {
Url = store.Url, DefaultFileSystem = store.DefaultFileSystem
};
var smugglerApi = new SmugglerFilesApi();
var export = await smugglerApi.ExportData(new SmugglerExportOptions <FilesConnectionStringOptions> {
From = options, ToFile = outputDirectory.ToFullPath()
});
await smugglerApi.ImportData(new SmugglerImportOptions <FilesConnectionStringOptions> {
FromFile = export.FilePath, To = options
});
await VerifyDump(store, export.FilePath, s =>
{
using (var session = s.OpenAsyncSession())
{
var files = s.AsyncFilesCommands.BrowseAsync().Result;
Assert.Equal(1, files.Count());
}
});
}
finally
{
IOExtensions.DeleteDirectory(outputDirectory);
}
}
}
public async Task CanDumpEmptyFileSystem()
{
using (var store = NewStore())
{
var server = GetServer();
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
try
{
// now perform full backup
var dumper = new SmugglerFilesApi {
Options = { Incremental = true }
};
var export = await dumper.ExportData(
new SmugglerExportOptions <FilesConnectionStringOptions>
{
ToFile = outputDirectory,
From = new FilesConnectionStringOptions
{
Url = server.Url,
DefaultFileSystem = store.DefaultFileSystem,
}
});
await VerifyDump(store, export.FilePath, s =>
{
using (var session = s.OpenAsyncSession())
{
var files = s.AsyncFilesCommands.BrowseAsync().Result;
Assert.Equal(0, files.Count());
}
});
}
finally
{
IOExtensions.DeleteDirectory(outputDirectory);
}
}
}
public async Task ShouldThrowIfFileSystemDoesNotExist()
{
using (var store = NewStore())
{
var server = GetServer();
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
try
{
var smugglerApi = new SmugglerFilesApi();
var options = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = "DoesNotExist"
};
var message = string.Format("Smuggler does not support file system creation (file system 'DoesNotExist' on server '{0}' must exist before running Smuggler).", store.Url);
var e = await AssertAsync.Throws <SmugglerException>(() => smugglerApi.ExportData(new SmugglerExportOptions <FilesConnectionStringOptions> {
ToFile = outputDirectory.ToFullPath(), From = options
}));
Assert.Equal(message, e.Message);
e = await AssertAsync.Throws <SmugglerException>(() => smugglerApi.ImportData(new SmugglerImportOptions <FilesConnectionStringOptions> {
FromFile = outputDirectory.ToFullPath(), To = options
}));
Assert.Equal(message, e.Message);
}
finally
{
IOExtensions.DeleteDirectory(outputDirectory);
}
}
}
public async Task MetadataIsPreserved()
{
using (var store = NewStore())
{
var server = GetServer();
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
try
{
var dumper = new SmugglerFilesApi {
Options = { Incremental = true }
};
FileHeader originalFile;
using (var session = store.OpenAsyncSession())
{
session.RegisterUpload("test1.file", CreateRandomFileStream(12800));
await session.SaveChangesAsync();
// content update after a metadata change
originalFile = await session.LoadFileAsync("test1.file");
originalFile.Metadata["Test"] = new RavenJValue("Value");
await session.SaveChangesAsync();
}
using (new FilesStore {
Url = server.Url
}.Initialize())
{
// now perform full backup
await dumper.ExportData(new SmugglerExportOptions <FilesConnectionStringOptions>
{
ToFile = outputDirectory,
From = new FilesConnectionStringOptions
{
Url = server.Url,
DefaultFileSystem = store.DefaultFileSystem,
}
});
}
await VerifyDump(store, outputDirectory, s =>
{
using (var session = s.OpenAsyncSession())
{
var file = session.LoadFileAsync("test1.file").Result;
Assert.Equal(originalFile.CreationDate, file.CreationDate);
Assert.Equal(originalFile.Directory, file.Directory);
Assert.Equal(originalFile.Extension, file.Extension);
Assert.Equal(originalFile.FullPath, file.FullPath);
Assert.Equal(originalFile.Name, file.Name);
Assert.Equal(originalFile.TotalSize, file.TotalSize);
Assert.Equal(originalFile.UploadedSize, file.UploadedSize);
Assert.Equal(originalFile.LastModified, file.LastModified);
Assert.True(file.Metadata.ContainsKey("Test"));
}
});
}
finally
{
IOExtensions.DeleteDirectory(outputDirectory);
}
}
}
public async Task ContentIsPreserved_MultipleDirectories()
{
using (var store = NewStore())
{
ReseedRandom(100); // Force a random distribution.
var server = GetServer();
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
try
{
var dumper = new SmugglerFilesApi {
Options = { Incremental = false }
};
var files = new Stream[10];
using (var session = store.OpenAsyncSession())
{
for (int i = 0; i < files.Length; i++)
{
files[i] = CreateRandomFileStream(100 * i + 12);
session.RegisterUpload(i + "/test.file", files[i]);
}
await session.SaveChangesAsync();
}
ExportFilesResult result;
using (new FilesStore {
Url = server.Url
}.Initialize())
{
// now perform full backup
result = await dumper.ExportData(new SmugglerExportOptions <FilesConnectionStringOptions>
{
ToFile = outputDirectory,
From = new FilesConnectionStringOptions
{
Url = server.Url,
DefaultFileSystem = store.DefaultFileSystem,
}
});
}
await VerifyDump(store, result.FilePath, s =>
{
for (int i = 0; i < files.Length; i++)
{
using (var session = s.OpenAsyncSession())
{
var file = session.LoadFileAsync(i + "/test.file").Result;
var stream = session.DownloadAsync(file).Result;
files[i].Position = 0;
Assert.Equal(files[i].GetMD5Hash(), stream.GetMD5Hash());
}
}
});
}
finally
{
IOExtensions.DeleteDirectory(outputDirectory);
}
}
}
public async Task CanHandleFilesExceptionsGracefully()
{
using (var store = NewStore())
{
var server = GetServer();
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
var alreadyReset = false;
var proxyPort = 8070;
var forwarder = new ProxyServer(ref proxyPort, server.Configuration.Port)
{
VetoTransfer = (totalRead, buffer) =>
{
if (alreadyReset == false && totalRead > 28000)
{
alreadyReset = true;
return(true);
}
return(false);
}
};
try
{
ReseedRandom(100); // Force a random distribution.
await InitializeWithRandomFiles(store, 20, 30);
// now perform full backup
var dumper = new SmugglerFilesApi {
Options = { Incremental = true }
};
ExportFilesResult exportResult = null;
try
{
// We will ensure this one will fail somewhere along the line.
exportResult = await dumper.ExportData(
new SmugglerExportOptions <FilesConnectionStringOptions>
{
ToFile = outputDirectory,
From = new FilesConnectionStringOptions
{
Url = "http://localhost:" + proxyPort,
DefaultFileSystem = store.DefaultFileSystem,
}
});
}
catch (SmugglerExportException inner)
{
exportResult = new ExportFilesResult
{
FilePath = inner.File
};
}
Assert.NotNull(exportResult);
Assert.True(!string.IsNullOrWhiteSpace(exportResult.FilePath));
// Continue with the incremental dump.
exportResult = await dumper.ExportData(
new SmugglerExportOptions <FilesConnectionStringOptions>
{
ToFile = outputDirectory,
From = new FilesConnectionStringOptions
{
Url = server.Url,
DefaultFileSystem = store.DefaultFileSystem,
}
});
// Import everything and verify all files are there.
await VerifyDump(store, outputDirectory, s =>
{
using (var session = s.OpenAsyncSession())
{
var files = s.AsyncFilesCommands.BrowseAsync().Result;
Assert.Equal(20, files.Count());
}
});
}
finally
{
forwarder.Dispose();
IOExtensions.DeleteDirectory(outputDirectory);
}
}
}
public async Task Foo()
{
{
#region smuggler_api_1
SmugglerFilesApi smugglerApi = new SmugglerFilesApi();
var exportOptions = new SmugglerExportOptions <FilesConnectionStringOptions>
{
ToFile = "dump.ravenfs",
From = new FilesConnectionStringOptions
{
Url = "http://localhost:8080",
DefaultFileSystem = "NorthwindFS"
}
};
var exportResult = await smugglerApi.ExportData(exportOptions);
#endregion
}
{
#region smuggler_api_2
SmugglerFilesApi smugglerApi = new SmugglerFilesApi();
var importOptions = new SmugglerImportOptions <FilesConnectionStringOptions>
{
FromFile = "dump.ravenfs",
To = new FilesConnectionStringOptions
{
Url = "http://localhost:8080",
DefaultFileSystem = "NewNorthwindFS"
}
};
await smugglerApi.ImportData(importOptions);
#endregion
}
{
#region smuggler_api_3
// export files
// from NorthwindFS file system
// found on http://localhost:8080
// and import them to NewNorthwindFS
// found on the same server
SmugglerFilesApi smugglerApi = new SmugglerFilesApi();
var betweenOptions = new SmugglerBetweenOptions <FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = "http://localhost:8080",
DefaultFileSystem = "NorthwindFS"
},
To = new FilesConnectionStringOptions
{
Url = "http://localhost:8080",
DefaultFileSystem = "NewNorthwindFS"
}
};
await smugglerApi.Between(betweenOptions);
#endregion
}
}
public async Task OnlyActiveContentIsPreserved_MultipleDirectories()
{
using (var store = NewStore())
{
var server = GetServer();
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
var smugglerApi = new SmugglerFilesApi();
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(SourceFilesystem);
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(DestinationFilesystem);
int total = 10;
var files = new string[total];
using (var session = store.OpenAsyncSession(SourceFilesystem))
{
for (int i = 0; i < total; i++)
{
files[i] = i + "/test.file";
session.RegisterUpload(files[i], CreateRandomFileStream(100 * i + 12));
}
await session.SaveChangesAsync();
}
int deletedFiles = 0;
var rnd = new Random();
using (var session = store.OpenAsyncSession(SourceFilesystem))
{
for (int i = 0; i < total; i++)
{
if (rnd.Next(2) == 0)
{
session.RegisterFileDeletion(files[i]);
deletedFiles++;
}
}
await session.SaveChangesAsync();
}
// now perform full backup
var dumper = new SmugglerFilesApi {
Options = { Incremental = true }
};
var export = await dumper.ExportData(
new SmugglerExportOptions <FilesConnectionStringOptions>
{
ToFile = outputDirectory,
From = new FilesConnectionStringOptions
{
Url = server.Url,
DefaultFileSystem = SourceFilesystem,
}
});
await VerifyDump(store, export.FilePath, s =>
{
using (var session = s.OpenAsyncSession())
{
int activeFiles = 0;
for (int i = 0; i < total; i++)
{
var file = session.LoadFileAsync(files[i]).Result;
if (file != null)
{
activeFiles++;
}
}
Assert.Equal(total - deletedFiles, activeFiles);
}
});
}
}
public async Task BetweenOperation_CanHandleFilesExceptionsGracefully()
{
using (var store = NewStore())
{
store.DefaultFileSystem = SourceFilesystem;
var server = GetServer();
var alreadyReset = false;
var forwarder = new ProxyServer(8070, server.Configuration.Port)
{
VetoTransfer = (totalRead, buffer) =>
{
if (alreadyReset == false && totalRead > 28000)
{
alreadyReset = true;
return(true);
}
return(false);
}
};
var smugglerApi = new SmugglerFilesApi();
var options = new SmugglerBetweenOptions <FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = "http://localhost:8070",
DefaultFileSystem = SourceFilesystem
},
To = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = DestinationFilesystem
}
};
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(SourceFilesystem);
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(DestinationFilesystem);
ReseedRandom(100); // Force a random distribution.
await InitializeWithRandomFiles(store, 20, 30);
Etag lastEtag = Etag.InvalidEtag;
try
{
await smugglerApi.Between(options);
}
catch (SmugglerExportException inner)
{
lastEtag = inner.LastEtag;
}
Assert.NotEqual(Etag.InvalidEtag, lastEtag);
await smugglerApi.Between(options);
using (var session = store.OpenAsyncSession(DestinationFilesystem))
{
var files = await session.Commands.BrowseAsync();
Assert.Equal(20, files.Count());
}
}
}
public async Task BetweenOperation_CanHandleFilesExceptionsGracefully()
{
using (var store = NewStore())
{
store.DefaultFileSystem = SourceFilesystem;
var server = GetServer();
var alreadyReset = false;
var port = 8070;
var forwarder = new ProxyServer(ref port, server.Configuration.Port)
{
VetoTransfer = (totalRead, buffer) =>
{
var s = Encoding.UTF8.GetString(buffer.Array, buffer.Offset, buffer.Count);
if (alreadyReset == false && totalRead > 28000 && !s.Contains("200 OK"))
{
alreadyReset = true;
return(true);
}
return(false);
}
};
try
{
var smugglerApi = new SmugglerFilesApi();
var options = new SmugglerBetweenOptions <FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = "http://localhost:" + port,
DefaultFileSystem = SourceFilesystem
},
To = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = DestinationFilesystem
}
};
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(SourceFilesystem);
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(DestinationFilesystem);
ReseedRandom(100); // Force a random distribution.
await InitializeWithRandomFiles(store, 20, 30);
Etag lastEtag = Etag.InvalidEtag;
try
{
await smugglerApi.Between(options);
Assert.False(true, "Expected error to happen during this Between operation, but it didn't happen :-(");
}
catch (SmugglerExportException inner)
{
lastEtag = inner.LastEtag;
}
Assert.NotEqual(Etag.InvalidEtag, lastEtag);
await smugglerApi.Between(options);
using (var session = store.OpenAsyncSession(DestinationFilesystem))
{
var files = await session.Commands.BrowseAsync();
Assert.Equal(20, files.Count());
}
}
finally
{
forwarder.Dispose();
server.Dispose();
}
}
}
public async Task BetweenOperation_OnlyActiveContentIsPreserved_MultipleDirectories()
{
using (var store = NewStore())
{
var server = GetServer();
var smugglerApi = new SmugglerFilesApi();
var options = new SmugglerBetweenOptions <FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = SourceFilesystem
},
To = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = DestinationFilesystem
}
};
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(SourceFilesystem);
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(DestinationFilesystem);
int total = 10;
var files = new string[total];
using (var session = store.OpenAsyncSession(SourceFilesystem))
{
for (int i = 0; i < total; i++)
{
files[i] = i + "/test.file";
session.RegisterUpload(files[i], CreateRandomFileStream(100 * i + 12));
}
await session.SaveChangesAsync();
}
int deletedFiles = 0;
var rnd = new Random();
using (var session = store.OpenAsyncSession(SourceFilesystem))
{
for (int i = 0; i < total; i++)
{
if (rnd.Next(2) == 0)
{
session.RegisterFileDeletion(files[i]);
deletedFiles++;
}
}
await session.SaveChangesAsync();
}
await smugglerApi.Between(options);
using (var session = store.OpenAsyncSession(DestinationFilesystem))
{
int activeFiles = 0;
for (int i = 0; i < total; i++)
{
var file = await session.LoadFileAsync(files[i]);
if (file != null)
{
activeFiles++;
}
}
Assert.Equal(total - deletedFiles, activeFiles);
}
}
}
public async Task SmugglerCanStripReplicationInformationDuringImport()
{
using (var store = NewStore())
{
var server = GetServer();
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
try
{
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync("N1");
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync("N2");
var content = new MemoryStream(new byte[] { 1, 2, 3 })
{
Position = 0
};
var commands = store.AsyncFilesCommands.ForFileSystem("N1");
await commands.UploadAsync("test.bin", content, new RavenJObject { { "test", "value" } });
var metadata = await commands.GetMetadataForAsync("test.bin");
var source1 = metadata[SynchronizationConstants.RavenSynchronizationSource];
var version1 = metadata[SynchronizationConstants.RavenSynchronizationVersion];
var history1 = metadata[SynchronizationConstants.RavenSynchronizationHistory] as RavenJArray;
Assert.NotNull(source1);
Assert.NotNull(version1);
Assert.NotNull(history1);
Assert.Empty(history1);
var smugglerApi = new SmugglerFilesApi(new SmugglerFilesOptions {
StripReplicationInformation = true
});
var export = await smugglerApi.ExportData(new SmugglerExportOptions <FilesConnectionStringOptions> {
From = new FilesConnectionStringOptions {
Url = store.Url, DefaultFileSystem = "N1"
}, ToFile = outputDirectory
});
await smugglerApi.ImportData(new SmugglerImportOptions <FilesConnectionStringOptions> {
FromFile = export.FilePath, To = new FilesConnectionStringOptions {
Url = store.Url, DefaultFileSystem = "N2"
}
});
commands = store.AsyncFilesCommands.ForFileSystem("N2");
metadata = await commands.GetMetadataForAsync("test.bin");
var source2 = metadata[SynchronizationConstants.RavenSynchronizationSource];
var version2 = metadata[SynchronizationConstants.RavenSynchronizationVersion];
var history2 = metadata[SynchronizationConstants.RavenSynchronizationHistory] as RavenJArray;
Assert.NotEqual(source1, source2);
Assert.Equal(version1, version2);
Assert.NotNull(history2);
Assert.Empty(history2);
}
finally
{
IOExtensions.DeleteDirectory(outputDirectory);
}
}
}
public async Task CanDisableVersioningDuringImport()
{
long fileCount;
string export = Path.Combine(NewDataPath("export_3219"), "Export");
using (var store = NewStore())
{
for (int i = 0; i < 10; i++)
{
await store.AsyncFilesCommands.UploadAsync("test-" + i, StringToStream("hello"));
}
var options = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = store.DefaultFileSystem
};
var smuggler = new SmugglerFilesApi();
await smuggler.ExportData(new SmugglerExportOptions <FilesConnectionStringOptions>
{
From = options,
ToFile = export
});
fileCount = (await store.AsyncFilesCommands.GetStatisticsAsync()).FileCount;
}
using (var store = NewStore(activeBundles: "Versioning", fileSystemName: "Import", index: 1))
{
await store.AsyncFilesCommands.Configuration.SetKeyAsync(VersioningUtil.DefaultConfigurationName, new VersioningConfiguration
{
Id = VersioningUtil.DefaultConfigurationName
});
var smuggler = new SmugglerFilesApi(new SmugglerFilesOptions()
{
ShouldDisableVersioningBundle = true
});
var options = new FilesConnectionStringOptions {
Url = store.Url, DefaultFileSystem = store.DefaultFileSystem
};
await smuggler.ImportData(new SmugglerImportOptions <FilesConnectionStringOptions> {
FromFile = export, To = options
});
var fileCountAfterImport = (await store.AsyncFilesCommands.GetStatisticsAsync()).FileCount;
Assert.Equal(fileCount, fileCountAfterImport);
// after import versioning should be active
await store.AsyncFilesCommands.UploadAsync("with-rev", StringToStream("rev"));
using (var session = store.OpenAsyncSession())
{
var revisions = await session.GetRevisionsForAsync("with-rev", 0, 10);
Assert.Equal(1, revisions.Length);
}
}
}
public static async Task AsyncMain()
{
//using (var store = new FilesStore()
//{
// Url = "http://localhost:8080",
// DefaultFileSystem = "FS1"
//}.Initialize())
//{
// var ms = new MemoryStream();
// var buffer = new byte[1024];
// for (int i = 0; i < buffer.Length; i++)
// {
// buffer[i] = (byte)i;
// }
// ms.Write(buffer, 0, buffer.Length);
// ms.Position = 0;
// await store.AsyncFilesCommands.UploadAsync("file.txt", ms);
// var tasks = new Task[50];
// for (var i = 0; i < 15000; i += tasks.Length)
// {
// var sessio = store.OpenAsyncSession();
// for (var j = 0; j < tasks.Length; j++)
// {
// var targetNAme = "file" + (i + j).ToString() + ".txt";
// tasks[j] = store.AsyncFilesCommands.CopyAsync("file.txt", targetNAme);
// }
// Task.WaitAll(tasks);
// Console.WriteLine(i);
// }
//}
//var sp = Stopwatch.StartNew();
//var smugglerApi = new SmugglerFilesApi();
//await smugglerApi.ExportData(exportOptions: new SmugglerExportOptions<FilesConnectionStringOptions>()
//{
// From = new FilesConnectionStringOptions()
// {
// DefaultFileSystem = "FS1",
// Url = "http://localhost:8080",
// },
// ToFile = "c:\\Temp\\export.ravendump",
//});
//Console.WriteLine(sp.ElapsedMilliseconds);
var sp = Stopwatch.StartNew();
try
{
var smugglerApi = new SmugglerFilesApi();
await smugglerApi.ImportData(importOptions : new SmugglerImportOptions <FilesConnectionStringOptions>()
{
To = new FilesConnectionStringOptions()
{
DefaultFileSystem = "FS2",
Url = "http://localhost:8080",
},
FromFile = "c:\\Temp\\export.ravendump",
});
}
catch (Exception ex)
{
Console.WriteLine(ex);
Console.WriteLine(ex.StackTrace);
}
Console.ReadLine();
Console.WriteLine(sp.ElapsedMilliseconds);
}
public async Task Foo()
{
{
#region smuggler_api_1
SmugglerFilesApi smugglerApi = new SmugglerFilesApi();
var exportOptions = new SmugglerExportOptions<FilesConnectionStringOptions>
{
ToFile = "dump.ravenfs",
From = new FilesConnectionStringOptions
{
Url = "http://localhost:8080",
DefaultFileSystem = "NorthwindFS"
}
};
var exportResult = await smugglerApi.ExportData(exportOptions);
#endregion
}
{
#region smuggler_api_2
SmugglerFilesApi smugglerApi = new SmugglerFilesApi();
var importOptions = new SmugglerImportOptions<FilesConnectionStringOptions>
{
FromFile = "dump.ravenfs",
To = new FilesConnectionStringOptions
{
Url = "http://localhost:8080",
DefaultFileSystem = "NewNorthwindFS"
}
};
await smugglerApi.ImportData(importOptions);
#endregion
}
{
#region smuggler_api_3
// export files
// from NorthwindFS file system
// found on http://localhost:8080
// and import them to NewNorthwindFS
// found on the same server
SmugglerFilesApi smugglerApi = new SmugglerFilesApi();
var betweenOptions = new SmugglerBetweenOptions<FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = "http://localhost:8080",
DefaultFileSystem = "NorthwindFS"
},
To = new FilesConnectionStringOptions
{
Url = "http://localhost:8080",
DefaultFileSystem = "NewNorthwindFS"
}
};
await smugglerApi.Between(betweenOptions);
#endregion
}
}
