public void Should_mark_file_to_be_resolved_using_current_strategy()
{
var differenceChunk = new MemoryStream();
var sw = new StreamWriter(differenceChunk);
sw.Write("Coconut is Stupid");
sw.Flush();
var sourceContent = SyncTestUtils.PrepareSourceStream(10);
sourceContent.Position = 0;
var destinationContent = new CombinedStream(differenceChunk, sourceContent);
var destinationClient = NewClient(0);
var sourceClient = NewClient(1);
var sourceMetadata = new RavenJObject
{
{"SomeTest-metadata", "some-value"}
};
var destinationMetadata = new RavenJObject
{
{"SomeTest-metadata", "shouldnt-be-overwritten"}
};
destinationClient.UploadAsync("test.txt", destinationMetadata, destinationContent).Wait();
sourceContent.Position = 0;
sourceClient.UploadAsync("test.txt", sourceMetadata, sourceContent).Wait();
var shouldBeConflict = sourceClient.Synchronization.StartAsync("test.txt", destinationClient).Result;
Assert.Equal("File test.txt is conflicted", shouldBeConflict.Exception.Message);
destinationClient.Synchronization.ResolveConflictAsync("test.txt", ConflictResolutionStrategy.CurrentVersion).Wait();
var result = destinationClient.Synchronization.StartAsync("test.txt", sourceClient).Result;
Assert.Equal(destinationContent.Length, result.BytesCopied + result.BytesTransfered);
// check if conflict resolution has been properly set on the source
string resultMd5;
using (var resultFileContent = new MemoryStream())
{
var metadata = sourceClient.DownloadAsync("test.txt", resultFileContent).Result;
Assert.Equal("shouldnt-be-overwritten", metadata.Value<string>("SomeTest-Metadata"));
resultFileContent.Position = 0;
resultMd5 = resultFileContent.GetMD5Hash();
resultFileContent.Position = 0;
}
destinationContent.Position = 0;
var destinationMd5 = destinationContent.GetMD5Hash();
sourceContent.Position = 0;
Assert.True(resultMd5 == destinationMd5);
}
public async Task CanWorkWithWinAuthEnabled()
{
var client = NewAsyncClient(enableAuthentication: true, credentials: new NetworkCredential(FactIfWindowsAuthenticationIsAvailable.Admin.UserName, FactIfWindowsAuthenticationIsAvailable.Admin.Password, FactIfWindowsAuthenticationIsAvailable.Admin.Domain));
var ms = new MemoryStream(new byte[1024 * 1024 * 10]);
await client.UploadAsync("/dir/ms.bin", ms);
ms.Position = 0;
await client.UploadAsync("/dir/ms.bin", ms);
var result = new MemoryStream();
(await client.DownloadAsync("/dir/ms.bin")).CopyTo(result);
ms.Position = 0;
result.Position = 0;
Assert.Equal(ms.GetMD5Hash(), result.GetMD5Hash());
await client.RenameAsync("/dir/ms.bin", "/dir/sm.bin");
var searchResults = await client.SearchOnDirectoryAsync("/dir");
Assert.Equal(1, searchResults.FileCount);
var metadata = await client.GetMetadataForAsync("/dir/sm.bin");
Assert.NotNull(metadata);
var folders = await client.GetDirectoriesAsync();
Assert.Equal(1, folders.Length);
var searchFields = await client.GetSearchFieldsAsync();
Assert.True(searchFields.Length > 0);
var guid = await client.GetServerIdAsync();
Assert.NotEqual(Guid.Empty, guid);
await client.UpdateMetadataAsync("/dir/sm.bin", new RavenJObject() { { "Meta", "Data" } });
var results = await client.SearchAsync("Meta:Data");
Assert.Equal(1, results.FileCount);
var stats = await client.GetStatisticsAsync();
Assert.Equal(1, stats.FileCount);
}
public async Task CanWorkWithWinAuthEnabled()
{
var client = NewClient(enableAuthentication: true, credentials: new NetworkCredential(username, password, domain));
var ms = new MemoryStream(new byte[]{1, 2, 4});
await client.UploadAsync("/dir/ms.bin", ms);
var result = new MemoryStream();
await client.DownloadAsync("/dir/ms.bin", result);
ms.Position = 0;
result.Position = 0;
Assert.Equal(ms.GetMD5Hash(), result.GetMD5Hash());
await client.RenameAsync("/dir/ms.bin", "/dir/sm.bin");
var searchResults = await client.GetFilesAsync("/dir");
Assert.Equal(1, searchResults.FileCount);
var metadata = await client.GetMetadataForAsync("/dir/sm.bin");
Assert.NotNull(metadata);
var folders = await client.GetFoldersAsync();
Assert.Equal(1, folders.Length);
var searchFields = await client.GetSearchFieldsAsync();
Assert.True(searchFields.Length > 0);
var guid = await client.GetServerId();
Assert.NotEqual(Guid.Empty, guid);
await client.UpdateMetadataAsync("/dir/sm.bin", new RavenJObject() { { "Meta", "Data" } });
var results = await client.SearchAsync("Meta:Data");
Assert.Equal(1, results.FileCount);
var stats = await client.StatsAsync();
Assert.Equal(1, stats.FileCount);
}
public async void Synchronize_file_with_different_beginning(int size)
{
var differenceChunk = new MemoryStream();
var sw = new StreamWriter(differenceChunk);
sw.Write("Coconut is Stupid");
sw.Flush();
var sourceContent = SyncTestUtils.PrepareSourceStream(size);
sourceContent.Position = 0;
var destinationContent = new CombinedStream(differenceChunk, sourceContent) {Position = 0};
var sourceClient = NewClient(0);
var destinationClient = NewClient(1);
var sourceMetadata = new RavenJObject
{
{"SomeTest-metadata", "some-value"}
};
var destinationMetadata = new RavenJObject
{
{"SomeTest-metadata", "should-be-overwritten"}
};
await destinationClient.UploadAsync("test.txt", destinationMetadata, destinationContent);
sourceContent.Position = 0;
await sourceClient.UploadAsync("test.txt", sourceMetadata, sourceContent);
var result = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.txt");
Assert.Equal(sourceContent.Length, result.BytesCopied + result.BytesTransfered);
string resultMd5;
using (var resultFileContent = new MemoryStream())
{
var metadata = destinationClient.DownloadAsync("test.txt", resultFileContent).Result;
// REVIEW: (Oren) The xxx-yyy-zzz headers are being transformed to: Xxx-Yyy-Zzz by the underlying implementation of HTTP Client. Is that OK? The old test was able to handle it "case insensitively".
Assert.Equal("some-value", metadata.Value<string>("SomeTest-Metadata"));
resultFileContent.Position = 0;
resultMd5 = resultFileContent.GetMD5Hash();
resultFileContent.Position = 0;
}
sourceContent.Position = 0;
var sourceMd5 = sourceContent.GetMD5Hash();
Assert.True(resultMd5 == sourceMd5);
}
public void Synchronize_file_with_different_beginning(int size)
{
var differenceChunk = new MemoryStream();
var sw = new StreamWriter(differenceChunk);
sw.Write("Coconut is Stupid");
sw.Flush();
var sourceContent = SyncTestUtils.PrepareSourceStream(size);
sourceContent.Position = 0;
var destinationContent = new CombinedStream(differenceChunk, sourceContent) {Position = 0};
var sourceClient = NewClient(0);
var destinationClient = NewClient(1);
var sourceMetadata = new NameValueCollection
{
{"SomeTest-metadata", "some-value"}
};
var destinationMetadata = new NameValueCollection
{
{"SomeTest-metadata", "should-be-overwritten"}
};
destinationClient.UploadAsync("test.txt", destinationMetadata, destinationContent).Wait();
sourceContent.Position = 0;
sourceClient.UploadAsync("test.txt", sourceMetadata, sourceContent).Wait();
var result = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.txt");
Assert.Equal(sourceContent.Length, result.BytesCopied + result.BytesTransfered);
string resultMd5;
using (var resultFileContent = new MemoryStream())
{
var metadata = destinationClient.DownloadAsync("test.txt", resultFileContent).Result;
Assert.Equal("some-value", metadata["SomeTest-metadata"]);
resultFileContent.Position = 0;
resultMd5 = resultFileContent.GetMD5Hash();
resultFileContent.Position = 0;
}
sourceContent.Position = 0;
var sourceMd5 = sourceContent.GetMD5Hash();
Assert.True(resultMd5 == sourceMd5);
}
public async Task uploading_file_multiple_times_must_not_throw_key_duplicate_exception_on_esent_and_concurrency_exception_on_voron(string storage)
{
var r = new Random(1);
var bytes = new byte[1024];
r.NextBytes(bytes);
var ms = new MemoryStream(bytes);
var expectedHash = ms.GetMD5Hash();
var client = NewAsyncClient(requestedStorage: storage);
for (int i = 0; i < 500; i++)
{
ms.Position = 0;
await client.UploadAsync("abc.bin", ms);
}
var stream = await client.DownloadAsync("abc.bin");
Assert.Equal(expectedHash, stream.GetMD5Hash());
}
public async Task Should_transfer_entire_file_even_if_rename_operation_was_performed()
{
var source = NewClient(0);
var destination = NewClient(1);
var fileContent = new MemoryStream(new byte[] {1, 2, 3});
await source.UploadAsync("test.bin", fileContent);
await source.RenameAsync("test.bin", "renamed.bin");
SyncTestUtils.TurnOnSynchronization(source, destination);
var destinationSyncResults = await source.Synchronization.SynchronizeDestinationsAsync();
Assert.Equal(1, destinationSyncResults.Length);
var reports = destinationSyncResults[0].Reports.ToArray();
Assert.Null(reports[0].Exception);
Assert.Equal(SynchronizationType.ContentUpdate, reports[0].Type);
Assert.Equal("renamed.bin", reports[0].FileName);
fileContent.Position = 0;
Assert.Equal(fileContent.GetMD5Hash(), destination.GetMetadataForAsync("renamed.bin").Result["Content-MD5"]);
}
public void ShouldWork()
{
var client = NewClient();
var tasks = new List<Task>(10);
// upload 10 files with the same content but different names concurrently
Assert.DoesNotThrow(
() =>
Parallel.For(0, 10, x => tasks.Add(client.UploadAsync("test" + x, new MemoryStream(new byte[] {1, 2, 3, 4, 5})))));
Task.WaitAll(tasks.ToArray());
var hash = new MemoryStream(new byte[] {1, 2, 3, 4, 5}).GetMD5Hash();
for (var i = 0; i < 10; i++)
{
var uploadedContent = new MemoryStream();
client.DownloadAsync("test" + i, uploadedContent).Wait();
uploadedContent.Position = 0;
Assert.Equal(hash, uploadedContent.GetMD5Hash());
}
}
public void Should_synchronize_to_all_destinations()
{
StartServerInstance(AddtitionalServerInstancePortNumber);
var sourceContent = SyncTestUtils.PrepareSourceStream(10000);
sourceContent.Position = 0;
var sourceClient = NewClient(0);
var destination1Client = NewClient(1);
var destination2Client = new RavenFileSystemClient(ServerAddress(AddtitionalServerInstancePortNumber));
var destination1Content = new RandomlyModifiedStream(sourceContent, 0.01);
sourceContent.Position = 0;
var destination2Content = new RandomlyModifiedStream(sourceContent, 0.01);
sourceContent.Position = 0;
destination1Client.UploadAsync("test.bin", destination1Content).Wait();
destination2Client.UploadAsync("test.bin", destination2Content).Wait();
sourceContent.Position = 0;
sourceClient.UploadAsync("test.bin", sourceContent).Wait();
sourceContent.Position = 0;
sourceClient.Config.SetConfig(SynchronizationConstants.RavenSynchronizationDestinations, new NameValueCollection
{
{
"url",
destination1Client
.ServerUrl
},
{
"url",
destination2Client
.ServerUrl
}
}).Wait();
var destinationSyncResults = sourceClient.Synchronization.SynchronizeDestinationsAsync().Result;
// we expect conflicts after first attempt of synchronization
Assert.Equal(2, destinationSyncResults.Length);
Assert.Equal("File test.bin is conflicted", destinationSyncResults[0].Reports.ToArray()[0].Exception.Message);
Assert.Equal("File test.bin is conflicted", destinationSyncResults[1].Reports.ToArray()[0].Exception.Message);
destination1Client.Synchronization.ResolveConflictAsync("test.bin", ConflictResolutionStrategy.RemoteVersion).Wait();
destination2Client.Synchronization.ResolveConflictAsync("test.bin", ConflictResolutionStrategy.RemoteVersion).Wait();
destinationSyncResults = sourceClient.Synchronization.SynchronizeDestinationsAsync().Result;
// check if reports match
Assert.Equal(2, destinationSyncResults.Length);
var result1 = destinationSyncResults[0].Reports.ToArray()[0];
Assert.Equal(sourceContent.Length, result1.BytesCopied + result1.BytesTransfered);
var result2 = destinationSyncResults[1].Reports.ToArray()[0];
Assert.Equal(sourceContent.Length, result2.BytesCopied + result2.BytesTransfered);
// check content of files
string destination1Md5;
using (var resultFileContent = new MemoryStream())
{
destination1Client.DownloadAsync("test.bin", resultFileContent).Wait();
resultFileContent.Position = 0;
destination1Md5 = resultFileContent.GetMD5Hash();
}
string destination2Md5;
using (var resultFileContent = new MemoryStream())
{
destination2Client.DownloadAsync("test.bin", resultFileContent).Wait();
resultFileContent.Position = 0;
destination2Md5 = resultFileContent.GetMD5Hash();
}
sourceContent.Position = 0;
var sourceMd5 = sourceContent.GetMD5Hash();
Assert.Equal(sourceMd5, destination1Md5);
Assert.Equal(sourceMd5, destination2Md5);
Assert.Equal(destination1Md5, destination2Md5);
}
public async Task After_file_delete_next_synchronization_should_override_tombsone()
{
var source = NewAsyncClient(0);
var destination = NewAsyncClient(1);
var sourceContent = new MemoryStream(new byte[] {5, 10, 15}) {Position = 0};
await source.UploadAsync("test.bin", sourceContent);
var report = await source.Synchronization.StartAsync("test.bin", destination);
Assert.Null(report.Exception);
await destination.DeleteAsync("test.bin");
report = await source.Synchronization.StartAsync("test.bin", destination);
Assert.Null(report.Exception);
var destContent = await destination.DownloadAsync("test.bin");
var destMetadata = await destination.GetMetadataForAsync("test.bin");
Assert.True(destMetadata[SynchronizationConstants.RavenDeleteMarker] == null, "Metadata should not containt Raven-Delete-Marker");
sourceContent.Position = 0;
Assert.Equal(sourceContent.GetMD5Hash(), destContent.GetMD5Hash());
}
public async Task Can_synchronize_file_that_doesnt_have_any_signature_while_file_on_destination_has()
{
const int size1B = 1;
const int size5Mb = 1024*1024*5;
var source = NewAsyncClient(0);
var destination = NewAsyncClient(1);
var buffer = new byte[size1B]; // 1b file should have no signatures
new Random().NextBytes(buffer);
var sourceContent = new MemoryStream(buffer);
await source.UploadAsync("test.bin", sourceContent);
buffer = new byte[size5Mb]; // 5Mb file should have 2 signatures
new Random().NextBytes(buffer);
await destination.UploadAsync("test.bin", new MemoryStream(buffer));
var sourceSigCount = source.Synchronization.GetRdcManifestAsync("test.bin").Result.Signatures.Count;
var destinationSigCount = destination.Synchronization.GetRdcManifestAsync("test.bin").Result.Signatures.Count;
Assert.Equal(0, sourceSigCount); // ensure that file on source has no signature
Assert.True(destinationSigCount > 0, "File on destination should have any signature");
var result = SyncTestUtils.ResolveConflictAndSynchronize(source, destination, "test.bin");
Assert.Null(result.Exception);
Assert.Equal(size1B, result.BytesTransfered);
sourceContent.Position = 0;
Assert.Equal(sourceContent.GetMD5Hash(), destination.GetMetadataForAsync("test.bin").Result.Value<string>("Content-MD5"));
}
public void Can_synchronize_file_with_less_number_of_signatures()
{
const int size5Mb = 1024*1024*5;
const int size1Mb = 1024*1024;
var source = NewAsyncClient(0);
var destination = NewAsyncClient(1);
var buffer = new byte[size1Mb]; // 1Mb file should have 1 signature
new Random().NextBytes(buffer);
var sourceContent = new MemoryStream(buffer);
source.UploadAsync("test.bin", sourceContent).Wait();
buffer = new byte[size5Mb]; // while 5Mb file has 2 signatures
new Random().NextBytes(buffer);
destination.UploadAsync("test.bin", new MemoryStream(buffer)).Wait();
var sourceSigCount = source.Synchronization.GetRdcManifestAsync("test.bin").Result.Signatures.Count;
var destinationSigCount = destination.Synchronization.GetRdcManifestAsync("test.bin").Result.Signatures.Count;
Assert.True(sourceSigCount > 0, "Source file should have one signature");
// ensure that file on source has less signatures than file on destination
Assert.True(sourceSigCount < destinationSigCount, "File on source should be smaller in order to have less signatures");
var result = SyncTestUtils.ResolveConflictAndSynchronize(source, destination, "test.bin");
Assert.Null(result.Exception);
Assert.Equal(size1Mb, result.BytesTransfered + result.BytesCopied);
sourceContent.Position = 0;
Assert.Equal(sourceContent.GetMD5Hash(), destination.GetMetadataForAsync("test.bin").Result.Value<string>("Content-MD5"));
}
public void Should_not_change_content_hash_after_metadata_upload()
{
var buffer = new byte[1024];
new Random().NextBytes(buffer);
var sourceContent = new MemoryStream(buffer);
var sourceClient = NewAsyncClient(0);
sourceClient.UploadAsync("test.bin", sourceContent).Wait();
sourceClient.UpdateMetadataAsync("test.bin", new RavenJObject { { "someKey", "someValue" } }).Wait();
sourceContent.Position = 0;
var resultFileMetadata = sourceClient.GetMetadataForAsync("test.bin").Result;
Assert.Contains("Content-MD5", resultFileMetadata.Keys);
Assert.Equal(sourceContent.GetMD5Hash(), resultFileMetadata.Value<string>("Content-MD5"));
}
public void Should_calculate_and_save_content_hash_after_synchronization()
{
var buffer = new byte[1024*1024*5 + 10];
new Random().NextBytes(buffer);
var sourceContent = new MemoryStream(buffer);
var sourceClient = NewAsyncClient(0);
sourceClient.UploadAsync("test.bin", sourceContent).Wait();
sourceContent.Position = 0;
var destinationClient = NewAsyncClient(1);
destinationClient.UploadAsync("test.bin", new RandomlyModifiedStream(sourceContent, 0.01)).Wait();
sourceContent.Position = 0;
SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.bin");
var resultFileMetadata = destinationClient.GetMetadataForAsync("test.bin").Result;
Assert.Contains("Content-MD5", resultFileMetadata.Keys);
Assert.Equal(sourceContent.GetMD5Hash(), resultFileMetadata.Value<string>("Content-MD5"));
}
public async Task Synchronization_of_already_synchronized_file_should_detect_that_no_work_is_needed(int size, int? seed)
{
Random r;
r = seed != null ? new Random(seed.Value) : new Random();
var bytes = new byte[size];
r.NextBytes(bytes);
var sourceContent = new MemoryStream(bytes);
var destinationContent = new RandomlyModifiedStream(new RandomStream(size, 1), 0.01, seed);
var destinationClient = NewAsyncClient(0);
var sourceClient = NewAsyncClient(1);
var srcMd5 = sourceContent.GetMD5Hash();
sourceContent.Position = 0;
var dstMd5 = (new RandomlyModifiedStream(new RandomStream(size, 1), 0.01, seed)).GetMD5Hash();
await destinationClient.UploadAsync("test.bin", destinationContent, new RavenJObject());
await sourceClient.UploadAsync("test.bin", sourceContent, new RavenJObject());
var firstSynchronization = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.bin");
Assert.Equal(sourceContent.Length, firstSynchronization.BytesCopied + firstSynchronization.BytesTransfered);
string resultMd5;
using (var resultFileContent = await destinationClient.DownloadAsync("test.bin"))
{
resultMd5 = resultFileContent.GetMD5Hash();
}
sourceContent.Position = 0;
var sourceMd5 = sourceContent.GetMD5Hash();
Assert.Equal(sourceMd5, resultMd5);
var secondSynchronization = sourceClient.Synchronization.StartAsync("test.bin", destinationClient).Result;
using (var resultFileContent = await destinationClient.DownloadAsync("test.bin"))
{
resultMd5 = resultFileContent.GetMD5Hash();
}
sourceContent.Position = 0;
sourceMd5 = sourceContent.GetMD5Hash();
Assert.Equal(sourceMd5, resultMd5);
Assert.Equal(0, secondSynchronization.NeedListLength);
Assert.Equal(0, secondSynchronization.BytesTransfered);
Assert.Equal(0, secondSynchronization.BytesCopied);
Assert.Equal("Destination server had this file in the past", secondSynchronization.Exception.Message);
}
public void Should_reuse_second_page_if_only_first_one_changed()
{
string filename = FileHeader.Canonize("test");
var file = SyncTestUtils.PreparePagesStream(2);
file.Position = 0;
var sourceContent = new MemoryStream();
file.CopyTo(sourceContent);
sourceContent.Position = 0;
sourceContent.Write(new byte[] {0, 0, 0, 0}, 0, 4); // change content of the 1st page
var destinationContent = file;
sourceContent.Position = 0;
source.UploadAsync(filename, sourceContent).Wait();
destinationContent.Position = 0;
destination.UploadAsync(filename, destinationContent).Wait();
var contentUpdate = new ContentUpdateWorkItem(filename, "http://localhost:12345", sourceRfs.Storage,
sourceRfs.SigGenerator);
sourceContent.Position = 0;
// force to upload entire file, we just want to check which pages will be reused
contentUpdate.UploadToAsync(destination.Synchronization).Wait();
destination.Synchronization.ResolveConflictAsync(filename, ConflictResolutionStrategy.RemoteVersion).Wait();
contentUpdate.UploadToAsync(destination.Synchronization).Wait();
FileAndPagesInformation fileAndPages = null;
destinationRfs.Storage.Batch(accessor => fileAndPages = accessor.GetFile(filename, 0, 256));
Assert.Equal(2, fileAndPages.Pages.Count);
Assert.Equal(3, fileAndPages.Pages[0].Id); // new page -> id == 3
Assert.Equal(2, fileAndPages.Pages[1].Id); // reused page -> id still == 2
sourceContent.Position = 0;
Assert.Equal(sourceContent.GetMD5Hash(), destination.GetMetadataForAsync(filename).Result["Content-MD5"]);
}
public async Task Should_reuse_pages_where_nothing_has_changed()
{
string filename = FileHeader.Canonize("test");
var file = SyncTestUtils.PreparePagesStream(3);
file.Position = 0;
var sourceContent = new MemoryStream();
file.CopyTo(sourceContent);
sourceContent.Position = StorageConstants.MaxPageSize + 1;
sourceContent.Write(new byte[] {0, 0, 0, 0}, 0, 4); // change content of the 2nd page
var destinationContent = file;
sourceContent.Position = 0;
await source.UploadAsync(filename, sourceContent);
destinationContent.Position = 0;
await destination.UploadAsync(filename, destinationContent);
var contentUpdate = new ContentUpdateWorkItem(filename, "http://localhost:12345", sourceRfs.Storage, sourceRfs.SigGenerator);
sourceContent.Position = 0;
// force to upload entire file, we just want to check which pages will be reused
await contentUpdate.UploadToAsync(destination.Synchronization);
await destination.Synchronization.ResolveConflictAsync(filename, ConflictResolutionStrategy.RemoteVersion);
await contentUpdate.UploadToAsync(destination.Synchronization);
FileAndPagesInformation fileAndPages = null;
destinationRfs.Storage.Batch(accessor => fileAndPages = accessor.GetFile(filename, 0, 256));
Assert.Equal(3, fileAndPages.Pages.Count);
Assert.Equal(1, fileAndPages.Pages[0].Id); // reused page
Assert.Equal(4, fileAndPages.Pages[1].Id); // new page -> id == 4
Assert.Equal(3, fileAndPages.Pages[2].Id); // reused page
sourceContent.Position = 0;
var metadata = await destination.GetMetadataForAsync(filename);
Assert.Equal(sourceContent.GetMD5Hash(), metadata.Value<string>("Content-MD5"));
}
public async Task File_content_change_should_be_propagated()
{
var buffer = new byte[1024*1024*2]; // 2 MB
new Random().NextBytes(buffer);
var content = new MemoryStream(buffer);
var changedContent = new RandomlyModifiedStream(content, 0.02);
var server1 = NewClient(0);
var server2 = NewClient(1);
var server3 = NewClient(2);
content.Position = 0;
await server1.UploadAsync("test.bin", new RavenJObject { { "test", "value" } }, content);
Assert.Equal(1, server1.StatsAsync().Result.FileCount);
SyncTestUtils.TurnOnSynchronization(server1, server2);
Assert.Null(server1.Synchronization.SynchronizeDestinationsAsync().Result[0].Exception);
Assert.Equal(1, server2.StatsAsync().Result.FileCount);
SyncTestUtils.TurnOnSynchronization(server2, server3);
Assert.Null(server2.Synchronization.SynchronizeDestinationsAsync().Result[0].Exception);
Assert.Equal(1, server3.StatsAsync().Result.FileCount);
SyncTestUtils.TurnOffSynchronization(server1);
content.Position = 0;
await server1.UploadAsync("test.bin", changedContent);
SyncTestUtils.TurnOnSynchronization(server1, server2);
var secondServer1Synchronization = await server1.Synchronization.SynchronizeDestinationsAsync();
Assert.Null(secondServer1Synchronization[0].Exception);
Assert.Equal(SynchronizationType.ContentUpdate, secondServer1Synchronization[0].Reports.ToArray()[0].Type);
var secondServer2Synchronization = await server2.Synchronization.SynchronizeDestinationsAsync();
Assert.Null(secondServer2Synchronization[0].Exception);
Assert.Equal(SynchronizationType.ContentUpdate, secondServer2Synchronization[0].Reports.ToArray()[0].Type);
// On all servers should have the same content of the file
string server1Md5;
using (var resultFileContent = new MemoryStream())
{
server1.DownloadAsync("test.bin", resultFileContent).Wait();
resultFileContent.Position = 0;
server1Md5 = resultFileContent.GetMD5Hash();
}
string server2Md5;
using (var resultFileContent = new MemoryStream())
{
server2.DownloadAsync("test.bin", resultFileContent).Wait();
resultFileContent.Position = 0;
server2Md5 = resultFileContent.GetMD5Hash();
}
string server3Md5;
using (var resultFileContent = new MemoryStream())
{
server3.DownloadAsync("test.bin", resultFileContent).Wait();
resultFileContent.Position = 0;
server3Md5 = resultFileContent.GetMD5Hash();
}
Assert.Equal(server1Md5, server2Md5);
Assert.Equal(server2Md5, server3Md5);
Assert.Equal(1, server1.StatsAsync().Result.FileCount);
Assert.Equal(1, server2.StatsAsync().Result.FileCount);
Assert.Equal(1, server3.StatsAsync().Result.FileCount);
}
public void Should_reuse_pages_where_nothing_has_changed()
{
var file = SyncTestUtils.PreparePagesStream(3);
file.Position = 0;
var sourceContent = new MemoryStream();
file.CopyTo(sourceContent);
sourceContent.Position = StorageConstants.MaxPageSize + 1;
sourceContent.Write(new byte[] {0, 0, 0, 0}, 0, 4); // change content of the 2nd page
var destinationContent = file;
sourceContent.Position = 0;
source.UploadAsync("test", sourceContent).Wait();
destinationContent.Position = 0;
destination.UploadAsync("test", destinationContent).Wait();
var contentUpdate = new ContentUpdateWorkItem("test", "http://localhost:12345", sourceRfs.Storage,
sourceRfs.SigGenerator);
sourceContent.Position = 0;
// force to upload entire file, we just want to check which pages will be reused
contentUpdate.UploadToAsync(destination.ServerUrl).Wait();
destination.Synchronization.ResolveConflictAsync("test", ConflictResolutionStrategy.RemoteVersion).Wait();
contentUpdate.UploadToAsync(destination.ServerUrl).Wait();
FileAndPages fileAndPages = null;
destinationRfs.Storage.Batch(accessor => fileAndPages = accessor.GetFile("test", 0, 256));
Assert.Equal(3, fileAndPages.Pages.Count);
Assert.Equal(1, fileAndPages.Pages[0].Id); // reused page
Assert.Equal(4, fileAndPages.Pages[1].Id); // new page -> id == 4
Assert.Equal(3, fileAndPages.Pages[2].Id); // reused page
sourceContent.Position = 0;
Assert.Equal(sourceContent.GetMD5Hash(), destination.GetMetadataForAsync("test").Result["Content-MD5"]);
}
public void Should_synchronize_to_all_destinations()
{
var sourceContent = SyncTestUtils.PrepareSourceStream(10000);
sourceContent.Position = 0;
var sourceClient = NewClient(0);
var destination1Client = NewClient(1);
var destination2Client = NewClient(2);
var destination1Content = new RandomlyModifiedStream(sourceContent, 0.01);
sourceContent.Position = 0;
var destination2Content = new RandomlyModifiedStream(sourceContent, 0.01);
sourceContent.Position = 0;
destination1Client.UploadAsync("test.bin", destination1Content).Wait();
destination2Client.UploadAsync("test.bin", destination2Content).Wait();
sourceContent.Position = 0;
sourceClient.UploadAsync("test.bin", sourceContent).Wait();
sourceContent.Position = 0;
sourceClient.Config.SetDestinationsConfig(destination1Client.ToSynchronizationDestination(), destination2Client.ToSynchronizationDestination()).Wait();
var destinationSyncResults = sourceClient.Synchronization.SynchronizeDestinationsAsync().Result;
// we expect conflicts after first attempt of synchronization
Assert.Equal(2, destinationSyncResults.Length);
Assert.Equal("File test.bin is conflicted", destinationSyncResults[0].Reports.ToArray()[0].Exception.Message);
Assert.Equal("File test.bin is conflicted", destinationSyncResults[1].Reports.ToArray()[0].Exception.Message);
destination1Client.Synchronization.ResolveConflictAsync("test.bin", ConflictResolutionStrategy.RemoteVersion).Wait();
destination2Client.Synchronization.ResolveConflictAsync("test.bin", ConflictResolutionStrategy.RemoteVersion).Wait();
destinationSyncResults = sourceClient.Synchronization.SynchronizeDestinationsAsync().Result;
var conflictItem = destination1Client.Config.GetConfig<ConflictItem>(RavenFileNameHelper.ConflictConfigNameForFile("test.bin")).Result;
Assert.Null(conflictItem);
conflictItem = destination2Client.Config.GetConfig<ConflictItem>(RavenFileNameHelper.ConflictConfigNameForFile("test.bin")).Result;
Assert.Null(conflictItem);
// check if reports match
Assert.Equal(2, destinationSyncResults.Length);
var result1 = destinationSyncResults[0].Reports.ToArray()[0];
Assert.Equal(sourceContent.Length, result1.BytesCopied + result1.BytesTransfered);
Assert.Equal(SynchronizationType.ContentUpdate, result1.Type);
var result2 = destinationSyncResults[1].Reports.ToArray()[0];
Assert.Equal(sourceContent.Length, result2.BytesCopied + result2.BytesTransfered);
Assert.Equal(SynchronizationType.ContentUpdate, result2.Type);
// check content of files
string destination1Md5;
using (var resultFileContent = new MemoryStream())
{
destination1Client.DownloadAsync("test.bin", resultFileContent).Wait();
resultFileContent.Position = 0;
destination1Md5 = resultFileContent.GetMD5Hash();
}
string destination2Md5;
using (var resultFileContent = new MemoryStream())
{
destination2Client.DownloadAsync("test.bin", resultFileContent).Wait();
resultFileContent.Position = 0;
destination2Md5 = resultFileContent.GetMD5Hash();
}
sourceContent.Position = 0;
var sourceMd5 = sourceContent.GetMD5Hash();
Assert.Equal(sourceMd5, destination1Md5);
Assert.Equal(sourceMd5, destination2Md5);
Assert.Equal(destination1Md5, destination2Md5);
}
public void Synchronize_file_with_appended_data(int size)
{
var differenceChunk = new MemoryStream();
var sw = new StreamWriter(differenceChunk);
sw.Write("Coconut is Stupid");
sw.Flush();
var sourceContent = new CombinedStream(SyncTestUtils.PrepareSourceStream(size), differenceChunk) {Position = 0};
var destinationContent = SyncTestUtils.PrepareSourceStream(size);
destinationContent.Position = 0;
var sourceClient = NewClient(0);
var destinationClient = NewClient(1);
destinationClient.UploadAsync("test.txt", destinationContent).Wait();
sourceContent.Position = 0;
sourceClient.UploadAsync("test.txt", sourceContent).Wait();
var result = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.txt");
Assert.Equal(sourceContent.Length, result.BytesCopied + result.BytesTransfered);
string resultMd5;
using (var resultFileContent = new MemoryStream())
{
destinationClient.DownloadAsync("test.txt", resultFileContent).Wait();
resultFileContent.Position = 0;
resultMd5 = resultFileContent.GetMD5Hash();
resultFileContent.Position = 0;
}
sourceContent.Position = 0;
var sourceMd5 = sourceContent.GetMD5Hash();
Assert.True(resultMd5 == sourceMd5);
}
public void Should_calculate_and_save_content_hash_after_upload()
{
var buffer = new byte[1024];
new Random().NextBytes(buffer);
var sourceContent = new MemoryStream(buffer);
var sourceClient = NewClient(0);
sourceClient.UploadAsync("test.bin", sourceContent).Wait();
sourceContent.Position = 0;
var resultFileMetadata = sourceClient.GetMetadataForAsync("test.bin").Result;
Assert.Contains("Content-MD5", resultFileMetadata.AllKeys);
Assert.Equal(sourceContent.GetMD5Hash(), resultFileMetadata["Content-MD5"]);
}
public void Should_have_the_same_content(int size)
{
var sourceContent = SyncTestUtils.PrepareSourceStream(size);
sourceContent.Position = 0;
var destinationContent = new RandomlyModifiedStream(sourceContent, 0.01);
var destinationClient = NewClient(0);
var sourceClient = NewClient(1);
destinationClient.UploadAsync("test.txt", new NameValueCollection(), destinationContent).Wait();
sourceContent.Position = 0;
sourceClient.UploadAsync("test.txt", new NameValueCollection(), sourceContent).Wait();
var result = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.txt");
Assert.Equal(sourceContent.Length, result.BytesCopied + result.BytesTransfered);
string resultMd5;
using (var resultFileContent = new MemoryStream())
{
destinationClient.DownloadAsync("test.txt", resultFileContent).Wait();
resultFileContent.Position = 0;
resultMd5 = resultFileContent.GetMD5Hash();
}
sourceContent.Position = 0;
var sourceMd5 = sourceContent.GetMD5Hash();
Assert.Equal(sourceMd5, resultMd5);
}
