public async Task ShouldThrowIfFileSystemDoesNotExist()
{
using (var store = NewStore())
{
var server = GetServer();
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
try
{
var smugglerApi = new SmugglerFilesApi();
var options = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = "DoesNotExist"
};
var message = string.Format("Smuggler does not support file system creation (file system 'DoesNotExist' on server '{0}' must exist before running Smuggler).", store.Url);
var e = await AssertAsync.Throws<SmugglerException>(() => smugglerApi.ExportData(new SmugglerExportOptions<FilesConnectionStringOptions> { ToFile = outputDirectory.ToFullPath(), From = options }));
Assert.Equal(message, e.Message);
e = await AssertAsync.Throws<SmugglerException>(() => smugglerApi.ImportData(new SmugglerImportOptions<FilesConnectionStringOptions> { FromFile = outputDirectory.ToFullPath(), To = options }));
Assert.Equal(message, e.Message);
}
finally
{
IOExtensions.DeleteDirectory(outputDirectory);
}
}
}
public async Task BetweenOperation_ShouldNotCreateFilesystem()
{
using (var store = NewStore())
{
var smugglerApi = new SmugglerFilesApi();
var options = new SmugglerBetweenOptions<FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = SourceFilesystem
},
To = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = DestinationFilesystem
}
};
var exception = await AssertAsync.Throws<SmugglerException>(() => smugglerApi.Between(options));
Assert.True(exception.Message.StartsWith("Smuggler does not support file system creation (file system '" + SourceFilesystem + "' on server"));
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(SourceFilesystem);
exception = await AssertAsync.Throws<SmugglerException>(() => smugglerApi.Between(options));
Assert.True(exception.Message.StartsWith("Smuggler does not support file system creation (file system '" + DestinationFilesystem + "' on server"));
}
}
public async Task CanDisableVersioningDuringImport()
{
long fileCount;
string export = Path.Combine(NewDataPath("export_3219"), "Export");
using (var store = NewStore())
{
for (int i = 0; i < 10; i++)
{
await store.AsyncFilesCommands.UploadAsync("test-" + i, StringToStream("hello"));
}
var options = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = store.DefaultFileSystem
};
var smuggler = new SmugglerFilesApi();
await smuggler.ExportData(new SmugglerExportOptions<FilesConnectionStringOptions>
{
From = options,
ToFile = export
});
fileCount = (await store.AsyncFilesCommands.GetStatisticsAsync()).FileCount;
}
using (var store = NewStore(activeBundles: "Versioning", fileSystemName: "Import", index: 1))
{
await store.AsyncFilesCommands.Configuration.SetKeyAsync(VersioningUtil.DefaultConfigurationName, new VersioningConfiguration
{
Id = VersioningUtil.DefaultConfigurationName
});
var smuggler = new SmugglerFilesApi(new SmugglerFilesOptions()
{
ShouldDisableVersioningBundle = true
});
var options = new FilesConnectionStringOptions { Url = store.Url, DefaultFileSystem = store.DefaultFileSystem };
await smuggler.ImportData(new SmugglerImportOptions<FilesConnectionStringOptions> { FromFile = export, To = options });
var fileCountAfterImport = (await store.AsyncFilesCommands.GetStatisticsAsync()).FileCount;
Assert.Equal(fileCount, fileCountAfterImport);
// after import versioning should be active
await store.AsyncFilesCommands.UploadAsync("with-rev", StringToStream("rev"));
using (var session = store.OpenAsyncSession())
{
var revisions = await session.GetRevisionsForAsync("with-rev", 0, 10);
Assert.Equal(1, revisions.Length);
}
}
}
public async Task SmugglerCanStripReplicationInformationDuringImport()
{
using (var store = NewStore())
{
var server = GetServer();
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
try
{
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync("N1");
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync("N2");
var content = new MemoryStream(new byte[] { 1, 2, 3 })
{
Position = 0
};
var commands = store.AsyncFilesCommands.ForFileSystem("N1");
await commands.UploadAsync("test.bin", content, new RavenJObject { { "test", "value" } });
var metadata = await commands.GetMetadataForAsync("test.bin");
var source1 = metadata[SynchronizationConstants.RavenSynchronizationSource];
var version1 = metadata[SynchronizationConstants.RavenSynchronizationVersion];
var history1 = metadata[SynchronizationConstants.RavenSynchronizationHistory] as RavenJArray;
Assert.NotNull(source1);
Assert.NotNull(version1);
Assert.NotNull(history1);
Assert.Empty(history1);
var smugglerApi = new SmugglerFilesApi(new SmugglerFilesOptions { StripReplicationInformation = true });
var export = await smugglerApi.ExportData(new SmugglerExportOptions<FilesConnectionStringOptions> { From = new FilesConnectionStringOptions { Url = store.Url, DefaultFileSystem = "N1" }, ToFile = outputDirectory });
await smugglerApi.ImportData(new SmugglerImportOptions<FilesConnectionStringOptions> { FromFile = export.FilePath, To = new FilesConnectionStringOptions { Url = store.Url, DefaultFileSystem = "N2" } });
commands = store.AsyncFilesCommands.ForFileSystem("N2");
metadata = await commands.GetMetadataForAsync("test.bin");
var source2 = metadata[SynchronizationConstants.RavenSynchronizationSource];
var version2 = metadata[SynchronizationConstants.RavenSynchronizationVersion];
var history2 = metadata[SynchronizationConstants.RavenSynchronizationHistory] as RavenJArray;
Assert.NotEqual(source1, source2);
Assert.Equal(version1, version2);
Assert.NotNull(history2);
Assert.Empty(history2);
}
finally
{
IOExtensions.DeleteDirectory(outputDirectory);
}
}
}
public async Task BetweenOperation_BehaviorWhenServerIsDown()
{
string dataDirectory = null;
try
{
using (var store = NewStore())
{
var server = GetServer();
var smugglerApi = new SmugglerFilesApi();
var options = new SmugglerBetweenOptions<FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = "http://localhost:8078/",
DefaultFileSystem = SourceFilesystem
},
To = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = DestinationFilesystem
}
};
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(SourceFilesystem);
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(DestinationFilesystem);
var e = await AssertAsync.Throws<SmugglerException>(() => smugglerApi.Between(options));
Assert.Contains("Smuggler encountered a connection problem:", e.Message);
options.From.Url = store.Url;
options.To.Url = "http://localhost:8078/";
e = await AssertAsync.Throws<SmugglerException>(() => smugglerApi.Between(options));
Assert.Contains("Smuggler encountered a connection problem:", e.Message);
server.Dispose();
}
}
finally
{
if (!string.IsNullOrWhiteSpace(dataDirectory))
IOExtensions.DeleteDirectory(dataDirectory);
}
}
public async Task ShouldNotThrowIfFileSystemExists()
{
using (var store = NewStore(fileSystemName: "DoesExists", requestedStorage: "esent"))
{
var server = GetServer();
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
try
{
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(store.DefaultFileSystem);
await InitializeWithRandomFiles(store, 2, 4);
var options = new FilesConnectionStringOptions { Url = store.Url, DefaultFileSystem = store.DefaultFileSystem };
var smugglerApi = new SmugglerFilesApi();
var export = await smugglerApi.ExportData(new SmugglerExportOptions<FilesConnectionStringOptions> { From = options, ToFile = outputDirectory });
await smugglerApi.ImportData(new SmugglerImportOptions<FilesConnectionStringOptions> { FromFile = export.FilePath, To = options });
}
finally
{
IOExtensions.DeleteDirectory(outputDirectory);
}
}
}
public async Task CanHandleFilesExceptionsGracefully()
{
using (var store = NewStore())
{
var server = GetServer();
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
var alreadyReset = false;
var proxyPort = 8070;
var forwarder = new ProxyServer(ref proxyPort, server.Configuration.Port)
{
VetoTransfer = (totalRead, buffer) =>
{
if (alreadyReset == false && totalRead > 28000)
{
alreadyReset = true;
return true;
}
return false;
}
};
try
{
ReseedRandom(100); // Force a random distribution.
await InitializeWithRandomFiles(store, 50, 30);
// now perform full backup
var dumper = new SmugglerFilesApi { Options = { Incremental = true } };
ExportFilesResult exportResult = null;
try
{
// We will ensure this one will fail somewhere along the line.
exportResult = await dumper.ExportData(
new SmugglerExportOptions<FilesConnectionStringOptions>
{
ToFile = outputDirectory,
From = new FilesConnectionStringOptions
{
Url = "http://localhost:" + proxyPort,
DefaultFileSystem = store.DefaultFileSystem,
}
});
}
catch (SmugglerExportException inner)
{
exportResult = new ExportFilesResult
{
FilePath = inner.File
};
}
Assert.NotNull(exportResult);
Assert.True(!string.IsNullOrWhiteSpace(exportResult.FilePath));
// Continue with the incremental dump.
exportResult = await dumper.ExportData(
new SmugglerExportOptions<FilesConnectionStringOptions>
{
ToFile = outputDirectory,
From = new FilesConnectionStringOptions
{
Url = server.Url,
DefaultFileSystem = store.DefaultFileSystem,
}
});
// Import everything and verify all files are there.
await VerifyDump(store, outputDirectory, s =>
{
using (var session = s.OpenAsyncSession())
{
var files = s.AsyncFilesCommands.BrowseAsync().Result;
Assert.Equal(50, files.Count());
}
});
}
finally
{
forwarder.Dispose();
IOExtensions.DeleteDirectory(outputDirectory);
}
}
}
public async Task OnlyActiveContentIsPreserved_MultipleDirectories()
{
using (var store = NewStore())
{
var server = GetServer();
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
var smugglerApi = new SmugglerFilesApi();
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(SourceFilesystem);
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(DestinationFilesystem);
int total = 10;
var files = new string[total];
using (var session = store.OpenAsyncSession(SourceFilesystem))
{
for (int i = 0; i < total; i++)
{
files[i] = i + "/test.file";
session.RegisterUpload(files[i], CreateRandomFileStream(100 * i + 12));
}
await session.SaveChangesAsync();
}
int deletedFiles = 0;
var rnd = new Random();
using (var session = store.OpenAsyncSession(SourceFilesystem))
{
for (int i = 0; i < total; i++)
{
if (rnd.Next(2) == 0)
{
session.RegisterFileDeletion(files[i]);
deletedFiles++;
}
}
await session.SaveChangesAsync();
}
// now perform full backup
var dumper = new SmugglerFilesApi { Options = { Incremental = true } };
var export = await dumper.ExportData(
new SmugglerExportOptions<FilesConnectionStringOptions>
{
ToFile = outputDirectory,
From = new FilesConnectionStringOptions
{
Url = server.Url,
DefaultFileSystem = SourceFilesystem,
}
});
await VerifyDump(store, export.FilePath, s =>
{
using (var session = s.OpenAsyncSession())
{
int activeFiles = 0;
for (int i = 0; i < total; i++)
{
var file = session.LoadFileAsync(files[i]).Result;
if (file != null)
activeFiles++;
}
Assert.Equal(total - deletedFiles, activeFiles);
}
});
}
}
public async Task BehaviorWhenServerIsDown()
{
using (var store = NewStore())
{
var server = GetServer();
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
try
{
var options = new FilesConnectionStringOptions { Url = "http://localhost:8078/", DefaultFileSystem = store.DefaultFileSystem };
var smugglerApi = new SmugglerFilesApi();
var e = await AssertAsync.Throws<SmugglerException>(() => smugglerApi.ImportData(new SmugglerImportOptions<FilesConnectionStringOptions> { FromFile = outputDirectory.ToFullPath(), To = options }));
Assert.Contains("Smuggler encountered a connection problem:", e.Message);
e = await AssertAsync.Throws<SmugglerException>(() => smugglerApi.ExportData(new SmugglerExportOptions<FilesConnectionStringOptions> { ToFile = outputDirectory.ToFullPath(), From = options }));
Assert.Contains("Smuggler encountered a connection problem:", e.Message);
}
finally
{
IOExtensions.DeleteDirectory(outputDirectory);
}
}
}
public async Task CanDumpEmptyFileSystem()
{
using (var store = NewStore())
{
var server = GetServer();
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
try
{
// now perform full backup
var dumper = new SmugglerFilesApi {Options = {Incremental = true}};
var export = await dumper.ExportData(
new SmugglerExportOptions<FilesConnectionStringOptions>
{
ToFile = outputDirectory,
From = new FilesConnectionStringOptions
{
Url = server.Url,
DefaultFileSystem = store.DefaultFileSystem,
}
});
await VerifyDump(store, export.FilePath, s =>
{
using (var session = s.OpenAsyncSession())
{
var files = s.AsyncFilesCommands.BrowseAsync().Result;
Assert.Equal(0, files.Count());
}
});
}
finally
{
IOExtensions.DeleteDirectory(outputDirectory);
}
}
}
public async Task BetweenOperation_CanHandleFilesExceptionsGracefully()
{
using (var store = NewStore())
{
store.DefaultFileSystem = SourceFilesystem;
var server = GetServer();
var alreadyReset = false;
var forwarder = new ProxyServer(8070, server.Configuration.Port)
{
VetoTransfer = (totalRead, buffer) =>
{
if (alreadyReset == false && totalRead > 28000)
{
alreadyReset = true;
return true;
}
return false;
}
};
var smugglerApi = new SmugglerFilesApi();
var options = new SmugglerBetweenOptions<FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = "http://localhost:8070",
DefaultFileSystem = SourceFilesystem
},
To = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = DestinationFilesystem
}
};
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(SourceFilesystem);
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(DestinationFilesystem);
ReseedRandom(100); // Force a random distribution.
await InitializeWithRandomFiles(store, 20, 30);
Etag lastEtag = Etag.InvalidEtag;
try
{
await smugglerApi.Between(options);
}
catch (SmugglerExportException inner)
{
lastEtag = inner.LastEtag;
}
Assert.NotEqual(Etag.InvalidEtag, lastEtag);
await smugglerApi.Between(options);
using (var session = store.OpenAsyncSession(DestinationFilesystem))
{
var files = await session.Commands.BrowseAsync();
Assert.Equal(20, files.Count());
}
}
}
public async Task CanDisableVersioningDuringImport_Between()
{
using (var storeExport = NewStore())
using (var storeImport = NewStore(activeBundles: "Versioning", fileSystemName: "Import", index: 1))
{
for (int i = 0; i < 10; i++)
{
await storeExport.AsyncFilesCommands.UploadAsync("test-" + i, StringToStream("hello"));
}
await storeImport.AsyncFilesCommands.Configuration.SetKeyAsync(VersioningUtil.DefaultConfigurationName, new VersioningConfiguration
{
Id = VersioningUtil.DefaultConfigurationName
});
var fileCount = (await storeExport.AsyncFilesCommands.GetStatisticsAsync()).FileCount;
var smuggler = new SmugglerFilesApi(new SmugglerFilesOptions()
{
ShouldDisableVersioningBundle = true
});
var options = new SmugglerBetweenOptions<FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = storeExport.Url,
DefaultFileSystem = storeExport.DefaultFileSystem
},
To = new FilesConnectionStringOptions
{
Url = storeImport.Url,
DefaultFileSystem = storeImport.DefaultFileSystem
}
};
await smuggler.Between(options);
var fileCountAfterImport = (await storeImport.AsyncFilesCommands.GetStatisticsAsync()).FileCount;
Assert.Equal(fileCount, fileCountAfterImport);
// after import versioning should be active
await storeImport.AsyncFilesCommands.UploadAsync("with-rev", StringToStream("rev"));
using (var session = storeImport.OpenAsyncSession())
{
var revisions = await session.GetRevisionsForAsync("with-rev", 0, 10);
Assert.Equal(1, revisions.Length);
}
}
}
private async Task VerifyDump(FilesStore store, string backupPath, Action<FilesStore> action)
{
try
{
store.DefaultFileSystem += "-Verify";
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(store.DefaultFileSystem);
bool incremental = false;
var directory = new DirectoryInfo(backupPath);
if ( directory.Exists )
incremental = true;
var dumper = new SmugglerFilesApi { Options = { Incremental = incremental } };
await dumper.ImportData(new SmugglerImportOptions<FilesConnectionStringOptions>
{
FromFile = backupPath,
To = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = store.DefaultFileSystem,
}
});
action(store);
}
catch (Exception e)
{
throw e.SimplifyException();
}
}
public async Task MetadataIsPreserved()
{
using (var store = NewStore())
{
var server = GetServer();
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
try
{
var dumper = new SmugglerFilesApi { Options = { Incremental = true } };
FileHeader originalFile;
using (var session = store.OpenAsyncSession())
{
session.RegisterUpload("test1.file", CreateRandomFileStream(12800));
await session.SaveChangesAsync();
// content update after a metadata change
originalFile = await session.LoadFileAsync("test1.file");
originalFile.Metadata["Test"] = new RavenJValue("Value");
await session.SaveChangesAsync();
}
using (new FilesStore { Url = server.Url }.Initialize())
{
// now perform full backup
await dumper.ExportData(new SmugglerExportOptions<FilesConnectionStringOptions>
{
ToFile = outputDirectory,
From = new FilesConnectionStringOptions
{
Url = server.Url,
DefaultFileSystem = store.DefaultFileSystem,
}
});
}
await VerifyDump(store, outputDirectory, s =>
{
using (var session = s.OpenAsyncSession())
{
var file = session.LoadFileAsync("test1.file").Result;
Assert.Equal(originalFile.CreationDate, file.CreationDate);
Assert.Equal(originalFile.Directory, file.Directory);
Assert.Equal(originalFile.Extension, file.Extension);
Assert.Equal(originalFile.FullPath, file.FullPath);
Assert.Equal(originalFile.Name, file.Name);
Assert.Equal(originalFile.TotalSize, file.TotalSize);
Assert.Equal(originalFile.UploadedSize, file.UploadedSize);
Assert.Equal(originalFile.LastModified, file.LastModified);
Assert.True(file.Metadata.ContainsKey("Test"));
}
});
}
finally
{
IOExtensions.DeleteDirectory(outputDirectory);
}
}
}
public async Task BetweenOperation_CanDumpEmptyFileSystem()
{
string dataDirectory = null;
try
{
using (var store = NewStore())
{
var server = GetServer();
dataDirectory = server.Configuration.DataDirectory;
var smugglerApi = new SmugglerFilesApi();
var options = new SmugglerBetweenOptions<FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = SourceFilesystem
},
To = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = DestinationFilesystem
}
};
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(SourceFilesystem);
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(DestinationFilesystem);
await smugglerApi.Between(options);
using (var session = store.OpenAsyncSession(DestinationFilesystem))
{
var files = await session.Commands.BrowseAsync();
Assert.Equal(0, files.Count());
}
server.Dispose();
}
}
finally
{
if (!string.IsNullOrWhiteSpace(dataDirectory))
IOExtensions.DeleteDirectory(dataDirectory);
}
}
public async Task BetweenOperation_ContentIsPreserved_SingleFile()
{
using (var store = NewStore())
{
var server = GetServer();
ReseedRandom(100); // Force a random distribution.
int fileSize = 10000;
var smugglerApi = new SmugglerFilesApi();
var options = new SmugglerBetweenOptions<FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = SourceFilesystem
},
To = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = DestinationFilesystem
}
};
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(SourceFilesystem);
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(DestinationFilesystem);
var fileContent = CreateRandomFileStream(fileSize);
using (var session = store.OpenAsyncSession(SourceFilesystem))
{
session.RegisterUpload("test1.file", fileContent);
await session.SaveChangesAsync();
}
await smugglerApi.Between(options);
fileContent.Position = 0;
using (var session = store.OpenAsyncSession(DestinationFilesystem))
{
var file = session.LoadFileAsync("test1.file").Result;
Assert.Equal(fileSize, file.TotalSize);
var stream = session.DownloadAsync(file).Result;
Assert.Equal(fileContent.GetMD5Hash(), stream.GetMD5Hash());
}
}
}
public async Task BetweenOperation_CanHandleFilesExceptionsGracefully()
{
using (var store = NewStore())
{
store.DefaultFileSystem = SourceFilesystem;
var server = GetServer();
var alreadyReset = false;
var port = 8070;
var forwarder = new ProxyServer(ref port, server.Configuration.Port)
{
VetoTransfer = (totalRead, buffer) =>
{
var s = Encoding.UTF8.GetString(buffer.Array, buffer.Offset, buffer.Count);
if (alreadyReset == false && totalRead > 28000 && !s.Contains("200 OK"))
{
alreadyReset = true;
return true;
}
return false;
}
};
try
{
var smugglerApi = new SmugglerFilesApi();
var options = new SmugglerBetweenOptions<FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = "http://localhost:" + port,
DefaultFileSystem = SourceFilesystem
},
To = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = DestinationFilesystem
}
};
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(SourceFilesystem);
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(DestinationFilesystem);
ReseedRandom(100); // Force a random distribution.
await InitializeWithRandomFiles(store, 20, 30);
Etag lastEtag = Etag.InvalidEtag;
try
{
await smugglerApi.Between(options);
Assert.False(true, "Expected error to happen during this Between operation, but it didn't happen :-(");
}
catch (SmugglerExportException inner)
{
lastEtag = inner.LastEtag;
}
Assert.NotEqual(Etag.InvalidEtag, lastEtag);
await smugglerApi.Between(options);
using (var session = store.OpenAsyncSession(DestinationFilesystem))
{
var files = await session.Commands.BrowseAsync();
Assert.Equal(20, files.Count());
}
}
finally
{
forwarder.Dispose();
server.Dispose();
}
}
}
public async Task ContentIsPreserved_MultipleDirectories()
{
using (var store = NewStore())
{
ReseedRandom(100); // Force a random distribution.
var server = GetServer();
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
try
{
var dumper = new SmugglerFilesApi { Options = { Incremental = false } };
var files = new Stream[10];
using (var session = store.OpenAsyncSession())
{
for (int i = 0; i < files.Length; i++)
{
files[i] = CreateRandomFileStream(100 * i + 12);
session.RegisterUpload(i + "/test.file", files[i]);
}
await session.SaveChangesAsync();
}
ExportFilesResult result;
using (new FilesStore { Url = server.Url }.Initialize())
{
// now perform full backup
result = await dumper.ExportData(new SmugglerExportOptions<FilesConnectionStringOptions>
{
ToFile = outputDirectory,
From = new FilesConnectionStringOptions
{
Url = server.Url,
DefaultFileSystem = store.DefaultFileSystem,
}
});
}
await VerifyDump(store, result.FilePath, s =>
{
for (int i = 0; i < files.Length; i++)
{
using (var session = s.OpenAsyncSession())
{
var file = session.LoadFileAsync(i + "/test.file").Result;
var stream = session.DownloadAsync(file).Result;
files[i].Position = 0;
Assert.Equal(files[i].GetMD5Hash(), stream.GetMD5Hash());
}
}
});
}
finally
{
IOExtensions.DeleteDirectory(outputDirectory);
}
}
}
public async Task BetweenOperation_ContentIsPreserved_MultipleDirectories()
{
using (var store = NewStore())
{
var server = GetServer();
var smugglerApi = new SmugglerFilesApi();
var options = new SmugglerBetweenOptions<FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = SourceFilesystem
},
To = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = DestinationFilesystem
}
};
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(SourceFilesystem);
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(DestinationFilesystem);
var files = new Stream[10];
using (var session = store.OpenAsyncSession(SourceFilesystem))
{
for (int i = 0; i < files.Length; i++)
{
files[i] = CreateRandomFileStream(100 * i + 12);
session.RegisterUpload(i + "/test.file", files[i]);
}
await session.SaveChangesAsync();
}
await smugglerApi.Between(options);
for (int i = 0; i < files.Length; i++)
{
using (var session = store.OpenAsyncSession(DestinationFilesystem))
{
var file = await session.LoadFileAsync(i + "/test.file");
var stream = await session.DownloadAsync(file);
files[i].Position = 0;
Assert.Equal(files[i].GetMD5Hash(), stream.GetMD5Hash());
}
}
}
}
public async Task BetweenOperation_OnlyActiveContentIsPreserved_MultipleDirectories()
{
using (var store = NewStore())
{
var server = GetServer();
var smugglerApi = new SmugglerFilesApi();
var options = new SmugglerBetweenOptions<FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = SourceFilesystem
},
To = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = DestinationFilesystem
}
};
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(SourceFilesystem);
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(DestinationFilesystem);
int total = 10;
var files = new string[total];
using (var session = store.OpenAsyncSession(SourceFilesystem))
{
for (int i = 0; i < total; i++)
{
files[i] = i + "/test.file";
session.RegisterUpload(files[i], CreateRandomFileStream(100 * i + 12));
}
await session.SaveChangesAsync();
}
int deletedFiles = 0;
var rnd = new Random();
using (var session = store.OpenAsyncSession(SourceFilesystem))
{
for (int i = 0; i < total; i++)
{
if (rnd.Next(2) == 0)
{
session.RegisterFileDeletion(files[i]);
deletedFiles++;
}
}
await session.SaveChangesAsync();
}
await smugglerApi.Between(options);
using (var session = store.OpenAsyncSession(DestinationFilesystem))
{
int activeFiles = 0;
for (int i = 0; i < total; i++)
{
var file = await session.LoadFileAsync(files[i]);
if (file != null)
activeFiles++;
}
Assert.Equal(total - deletedFiles, activeFiles);
}
}
}
public async Task BetweenOperation_MetadataIsPreserved()
{
using (var store = NewStore())
{
var server = GetServer();
var smugglerApi = new SmugglerFilesApi();
var options = new SmugglerBetweenOptions<FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = SourceFilesystem
},
To = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = DestinationFilesystem
}
};
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(SourceFilesystem);
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(DestinationFilesystem);
FileHeader originalFile;
using (var session = store.OpenAsyncSession(SourceFilesystem))
{
session.RegisterUpload("test1.file", CreateRandomFileStream(12800));
await session.SaveChangesAsync();
// content update after a metadata change
originalFile = await session.LoadFileAsync("test1.file");
originalFile.Metadata["Test"] = new RavenJValue("Value");
await session.SaveChangesAsync();
}
await smugglerApi.Between(options);
using (var session = store.OpenAsyncSession(DestinationFilesystem))
{
var file = await session.LoadFileAsync("test1.file");
Assert.Equal(originalFile.CreationDate, file.CreationDate);
Assert.Equal(originalFile.Directory, file.Directory);
Assert.Equal(originalFile.Extension, file.Extension);
Assert.Equal(originalFile.FullPath, file.FullPath);
Assert.Equal(originalFile.Name, file.Name);
Assert.Equal(originalFile.TotalSize, file.TotalSize);
Assert.Equal(originalFile.UploadedSize, file.UploadedSize);
Assert.Equal(originalFile.LastModified, file.LastModified);
Assert.True(file.Metadata.ContainsKey("Test"));
}
}
}
public async Task ShouldNotThrowIfFileSystemExistsUsingDefaultConfiguration()
{
using (var store = NewStore())
{
var server = GetServer();
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
try
{
await store.AsyncFilesCommands.Admin.EnsureFileSystemExistsAsync(store.DefaultFileSystem);
await InitializeWithRandomFiles(store, 1, 100);
var options = new FilesConnectionStringOptions {Url = store.Url, DefaultFileSystem = store.DefaultFileSystem};
var smugglerApi = new SmugglerFilesApi();
var export = await smugglerApi.ExportData(new SmugglerExportOptions<FilesConnectionStringOptions> { From = options, ToFile = outputDirectory.ToFullPath() });
await smugglerApi.ImportData(new SmugglerImportOptions<FilesConnectionStringOptions> { FromFile = export.FilePath, To = options });
await VerifyDump(store, export.FilePath, s =>
{
using (var session = s.OpenAsyncSession())
{
var files = s.AsyncFilesCommands.BrowseAsync().Result;
Assert.Equal(1, files.Count());
}
});
}
finally
{
IOExtensions.DeleteDirectory(outputDirectory);
}
}
}
public async Task ShouldSmuggleConfigurationsInBetweenOperation()
{
using (var exportStore = NewStore())
using (var importStore = NewStore(1))
{
for (int i = 0; i < 100; i++)
{
await exportStore.AsyncFilesCommands.Configuration.SetKeyAsync("items/" + i, new RavenJObject
{
{
"test", "value"
},
{
"test-array", new RavenJArray
{
"item-1", "item-2", "item-3"
}
}
});
}
var countOfConfigurations = (await exportStore.AsyncFilesCommands.Configuration.GetKeyNamesAsync(0, 200)).Length;
var options = new SmugglerBetweenOptions<FilesConnectionStringOptions>()
{
From = new FilesConnectionStringOptions()
{
Url = exportStore.Url,
DefaultFileSystem = exportStore.DefaultFileSystem
},
To = new FilesConnectionStringOptions()
{
Url = importStore.Url,
DefaultFileSystem = importStore.DefaultFileSystem
}
};
var smugglerFilesApi = new SmugglerFilesApi();
smugglerFilesApi.Options.BatchSize = 5;
await smugglerFilesApi.Between(options);
Assert.Equal(countOfConfigurations, (await importStore.AsyncFilesCommands.Configuration.GetKeyNamesAsync(0, 200)).Length);
for (int i = 0; i < 100; i++)
{
Assert.NotNull(await importStore.AsyncFilesCommands.Configuration.GetKeyAsync<RavenJObject>("items/" + i));
}
}
}
