public void Should_reuse_second_page_if_only_first_one_changed()
{
var file = SyncTestUtils.PreparePagesStream(2);
file.Position = 0;
var sourceContent = new MemoryStream();
file.CopyTo(sourceContent);
sourceContent.Position = 0;
sourceContent.Write(new byte[] { 0, 0, 0, 0 }, 0, 4); // change content of the 1st page
var destinationContent = file;
sourceContent.Position = 0;
source.UploadAsync("test", sourceContent).Wait();
destinationContent.Position = 0;
destination.UploadAsync("test", destinationContent).Wait();
var contentUpdate = new ContentUpdateWorkItem("test", "http://localhost:12345", sourceRfs.Storage,
sourceRfs.SigGenerator);
sourceContent.Position = 0;
// force to upload entire file, we just want to check which pages will be reused
contentUpdate.UploadToAsync(destination.Synchronization).Wait();
destination.Synchronization.ResolveConflictAsync("test", ConflictResolutionStrategy.RemoteVersion).Wait();
contentUpdate.UploadToAsync(destination.Synchronization).Wait();
FileAndPagesInformation fileAndPages = null;
destinationRfs.Storage.Batch(accessor => fileAndPages = accessor.GetFile("test", 0, 256));
Assert.Equal(2, fileAndPages.Pages.Count);
Assert.Equal(3, fileAndPages.Pages[0].Id); // new page -> id == 3
Assert.Equal(2, fileAndPages.Pages[1].Id); // reused page -> id still == 2
sourceContent.Position = 0;
Assert.Equal(sourceContent.GetMD5Hash(), destination.GetMetadataForAsync("test").Result["Content-MD5"]);
}
public void Should_get_all_finished_synchronizations()
{
var destinationClient = NewClient(0);
var sourceClient = NewClient(1);
var files = new[] { "test1.bin", "test2.bin", "test3.bin" };
// make sure that returns empty list if there are no finished synchronizations yet
var result = destinationClient.Synchronization.GetFinishedAsync().Result;
Assert.Equal(0, result.TotalCount);
foreach (var item in files)
{
var sourceContent = new MemoryStream();
var sw = new StreamWriter(sourceContent);
sw.Write("abc123");
sw.Flush();
sourceContent.Position = 0;
var destinationContent = new MemoryStream();
var sw2 = new StreamWriter(destinationContent);
sw2.Write("cba321");
sw2.Flush();
destinationContent.Position = 0;
Task.WaitAll(
destinationClient.UploadAsync(item, destinationContent),
sourceClient.UploadAsync(item, sourceContent));
SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, item);
}
result = destinationClient.Synchronization.GetFinishedAsync().Result;
Assert.Equal(files.Length, result.TotalCount);
}
public void Big_file_test(long size)
{
var sourceContent = new RandomStream(size);
var destinationContent = new RandomlyModifiedStream(new RandomStream(size), 0.01);
var destinationClient = NewClient(0);
var sourceClient = NewClient(1);
var sourceMetadata = new RavenJObject
{
{ "SomeTest-metadata", "some-value" }
};
var destinationMetadata = new RavenJObject
{
{ "SomeTest-metadata", "should-be-overwritten" }
};
destinationClient.UploadAsync("test.bin", destinationMetadata, destinationContent).Wait();
sourceClient.UploadAsync("test.bin", sourceMetadata, sourceContent).Wait();
SynchronizationReport result = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.bin");
Assert.Equal(sourceContent.Length, result.BytesCopied + result.BytesTransfered);
}
public async void Synchronize_file_with_appended_data(int size)
{
var differenceChunk = new MemoryStream();
var sw = new StreamWriter(differenceChunk);
sw.Write("Coconut is Stupid");
sw.Flush();
var sourceContent = new CombinedStream(SyncTestUtils.PrepareSourceStream(size), differenceChunk)
{
Position = 0
};
var destinationContent = SyncTestUtils.PrepareSourceStream(size);
destinationContent.Position = 0;
var sourceClient = NewAsyncClient(0);
var destinationClient = NewAsyncClient(1);
await destinationClient.UploadAsync("test.txt", destinationContent);
sourceContent.Position = 0;
await sourceClient.UploadAsync("test.txt", sourceContent);
var result = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.txt");
Assert.Equal(sourceContent.Length, result.BytesCopied + result.BytesTransfered);
string resultMd5;
using (var resultFileContent = await destinationClient.DownloadAsync("test.txt"))
{
resultMd5 = resultFileContent.GetMD5Hash();
}
sourceContent.Position = 0;
var sourceMd5 = sourceContent.GetMD5Hash();
Assert.True(resultMd5 == sourceMd5);
}
public void Should_calculate_and_save_content_hash_after_synchronization()
{
var buffer = new byte[1024 * 1024 * 5 + 10];
new Random().NextBytes(buffer);
var sourceContent = new MemoryStream(buffer);
var sourceClient = NewClient(0);
sourceClient.UploadAsync("test.bin", sourceContent).Wait();
sourceContent.Position = 0;
var destinationClient = NewClient(1);
destinationClient.UploadAsync("test.bin", new RandomlyModifiedStream(sourceContent, 0.01)).Wait();
sourceContent.Position = 0;
SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.bin");
var resultFileMetadata = destinationClient.GetMetadataForAsync("test.bin").Result;
Assert.Contains("Content-MD5", resultFileMetadata.Keys);
Assert.Equal(sourceContent.GetMD5Hash(), resultFileMetadata.Value <string>("Content-MD5"));
}
public async void Should_reuse_pages_when_data_appended(int numberOfPages)
{
var file = SyncTestUtils.PreparePagesStream(numberOfPages);
var sourceContent = new CombinedStream(file, SyncTestUtils.PreparePagesStream(numberOfPages));
// add new pages at the end
var destinationContent = file;
sourceContent.Position = 0;
source.UploadAsync("test", sourceContent).Wait();
destinationContent.Position = 0;
destination.UploadAsync("test", destinationContent).Wait();
var contentUpdate = new ContentUpdateWorkItem("test", "http://localhost:12345", sourceRfs.Storage, sourceRfs.SigGenerator);
// force to upload entire file, we just want to check which pages will be reused
await contentUpdate.UploadToAsync(destination.Synchronization);
await destination.Synchronization.ResolveConflictAsync("test", ConflictResolutionStrategy.RemoteVersion);
await contentUpdate.UploadToAsync(destination.Synchronization);
FileAndPagesInformation fileAndPages = null;
destinationRfs.Storage.Batch(accessor => fileAndPages = accessor.GetFile("test", 0, 2 * numberOfPages));
Assert.Equal(2 * numberOfPages, fileAndPages.Pages.Count);
for (var i = 0; i < numberOfPages; i++)
{
Assert.Equal(i + 1, fileAndPages.Pages[i].Id);
// if page ids are in the original order it means that they were used the existing pages
}
sourceContent.Position = 0;
Assert.Equal(sourceContent.GetMD5Hash(), destination.GetMetadataForAsync("test").Result["Content-MD5"]);
}
public async void Should_synchronize_to_all_destinations()
{
var sourceContent = SyncTestUtils.PrepareSourceStream(10000);
sourceContent.Position = 0;
var sourceClient = NewAsyncClient(0);
var destination1Client = NewAsyncClient(1);
var destination2Client = NewAsyncClient(2);
var destination1Content = new RandomlyModifiedStream(sourceContent, 0.01);
sourceContent.Position = 0;
var destination2Content = new RandomlyModifiedStream(sourceContent, 0.01);
sourceContent.Position = 0;
await destination1Client.UploadAsync("test.bin", destination1Content);
await destination2Client.UploadAsync("test.bin", destination2Content);
sourceContent.Position = 0;
await sourceClient.UploadAsync("test.bin", sourceContent);
sourceContent.Position = 0;
sourceClient.Synchronization.SetDestinationsAsync(destination1Client.ToSynchronizationDestination(), destination2Client.ToSynchronizationDestination()).Wait();
var destinationSyncResults = sourceClient.Synchronization.SynchronizeAsync().Result;
// we expect conflicts after first attempt of synchronization
Assert.Equal(2, destinationSyncResults.Length);
Assert.Equal("File test.bin is conflicted", destinationSyncResults[0].Reports.ToArray()[0].Exception.Message);
Assert.Equal("File test.bin is conflicted", destinationSyncResults[1].Reports.ToArray()[0].Exception.Message);
await destination1Client.Synchronization.ResolveConflictAsync("test.bin", ConflictResolutionStrategy.RemoteVersion);
await destination2Client.Synchronization.ResolveConflictAsync("test.bin", ConflictResolutionStrategy.RemoteVersion);
destinationSyncResults = await sourceClient.Synchronization.SynchronizeAsync();
var conflictItem = await destination1Client.Configuration.GetKeyAsync <ConflictItem>(RavenFileNameHelper.ConflictConfigNameForFile("test.bin"));
Assert.Null(conflictItem);
conflictItem = await destination2Client.Configuration.GetKeyAsync <ConflictItem>(RavenFileNameHelper.ConflictConfigNameForFile("test.bin"));
Assert.Null(conflictItem);
// check if reports match
Assert.Equal(2, destinationSyncResults.Length);
var result1 = destinationSyncResults[0].Reports.ToArray()[0];
Assert.Equal(sourceContent.Length, result1.BytesCopied + result1.BytesTransfered);
Assert.Equal(SynchronizationType.ContentUpdate, result1.Type);
var result2 = destinationSyncResults[1].Reports.ToArray()[0];
Assert.Equal(sourceContent.Length, result2.BytesCopied + result2.BytesTransfered);
Assert.Equal(SynchronizationType.ContentUpdate, result2.Type);
// check content of files
string destination1Md5;
using (var resultFileContent = await destination1Client.DownloadAsync("test.bin"))
{
destination1Md5 = resultFileContent.GetMD5Hash();
}
string destination2Md5;
using (var resultFileContent = await destination2Client.DownloadAsync("test.bin"))
{
destination2Md5 = resultFileContent.GetMD5Hash();
}
sourceContent.Position = 0;
var sourceMd5 = sourceContent.GetMD5Hash();
Assert.Equal(sourceMd5, destination1Md5);
Assert.Equal(sourceMd5, destination2Md5);
Assert.Equal(destination1Md5, destination2Md5);
}
public void Synchronization_of_already_synchronized_file_should_detect_that_no_work_is_needed(int size, int?seed)
{
Random r;
r = seed != null ? new Random(seed.Value) : new Random();
var bytes = new byte[size];
r.NextBytes(bytes);
var sourceContent = new MemoryStream(bytes);
var destinationContent = new RandomlyModifiedStream(new RandomStream(size, 1), 0.01, seed);
var destinationClient = NewClient(0);
var sourceClient = NewClient(1);
var srcMd5 = sourceContent.GetMD5Hash();
sourceContent.Position = 0;
var dstMd5 = (new RandomlyModifiedStream(new RandomStream(size, 1), 0.01, seed)).GetMD5Hash();
destinationClient.UploadAsync("test.bin", new RavenJObject(), destinationContent).Wait();
sourceClient.UploadAsync("test.bin", new RavenJObject(), sourceContent).Wait();
var firstSynchronization = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.bin");
Assert.Equal(sourceContent.Length, firstSynchronization.BytesCopied + firstSynchronization.BytesTransfered);
string resultMd5;
using (var resultFileContent = new MemoryStream())
{
destinationClient.DownloadAsync("test.bin", resultFileContent).Wait();
resultFileContent.Position = 0;
resultMd5 = resultFileContent.GetMD5Hash();
}
sourceContent.Position = 0;
var sourceMd5 = sourceContent.GetMD5Hash();
Assert.Equal(sourceMd5, resultMd5);
var secondSynchronization = sourceClient.Synchronization.StartAsync("test.bin", destinationClient).Result;
using (var resultFileContent = new MemoryStream())
{
destinationClient.DownloadAsync("test.bin", resultFileContent).Wait();
resultFileContent.Position = 0;
resultMd5 = resultFileContent.GetMD5Hash();
}
sourceContent.Position = 0;
sourceMd5 = sourceContent.GetMD5Hash();
Assert.Equal(sourceMd5, resultMd5);
Assert.Equal(0, secondSynchronization.NeedListLength);
Assert.Equal(0, secondSynchronization.BytesTransfered);
Assert.Equal(0, secondSynchronization.BytesCopied);
Assert.Equal("Destination server had this file in the past", secondSynchronization.Exception.Message);
}
public async Task File_content_change_should_be_propagated()
{
this.SynchronizationInterval = TimeSpan.FromSeconds(10);
var generator = new Random(1234);
var buffer = new byte[1024 * 1024 * 2]; // 2 MB
generator.NextBytes(buffer);
buffer[0] = 0;
var content = new MemoryStream(buffer);
var smallerBuffer = new byte[1024 * 1024];
generator.NextBytes(smallerBuffer);
smallerBuffer[0] = 1;
var changedContent = new MemoryStream(smallerBuffer);
var store1 = NewStore(0, fiddler: true);
var store2 = NewStore(1, fiddler: true);
var store3 = NewStore(2, fiddler: true);
var server1 = store1.AsyncFilesCommands;
var server2 = store2.AsyncFilesCommands;
var server3 = store3.AsyncFilesCommands;
content.Position = 0;
await server1.UploadAsync("test.bin", content, new RavenJObject { { "test", "value" } });
Assert.Equal(1, server1.GetStatisticsAsync().Result.FileCount);
SyncTestUtils.TurnOnSynchronization(server1, server2);
Assert.Null(server1.Synchronization.SynchronizeAsync().Result[0].Exception);
Assert.Equal(1, server2.GetStatisticsAsync().Result.FileCount);
SyncTestUtils.TurnOnSynchronization(server2, server3);
Assert.Null(server2.Synchronization.SynchronizeAsync().Result[0].Exception);
Assert.Equal(1, server3.GetStatisticsAsync().Result.FileCount);
SyncTestUtils.TurnOffSynchronization(server1);
content.Position = 0;
await server1.UploadAsync("test.bin", changedContent);
var syncTaskServer2 = store2.Changes()
.ForSynchronization()
.Where(x => x.Action == SynchronizationAction.Finish)
.Timeout(TimeSpan.FromSeconds(50))
.Take(1)
.ToTask();
var syncTaskServer3 = store3.Changes()
.ForSynchronization()
.Where(x => x.Action == SynchronizationAction.Finish)
.Timeout(TimeSpan.FromSeconds(50))
.Take(1)
.ToTask();
SyncTestUtils.TurnOnSynchronization(server1, server2);
var secondServer1Synchronization = await server1.Synchronization.SynchronizeAsync();
Assert.Null(secondServer1Synchronization[0].Exception);
Assert.Equal(SynchronizationType.ContentUpdate, secondServer1Synchronization[0].Reports.ToArray()[0].Type);
await syncTaskServer2;
await syncTaskServer3;
// On all servers should have the same content of the file
string server1Md5;
using (var resultFileContent = await server1.DownloadAsync("test.bin"))
{
server1Md5 = resultFileContent.GetMD5Hash();
}
string server2Md5;
using (var resultFileContent = await server2.DownloadAsync("test.bin"))
{
server2Md5 = resultFileContent.GetMD5Hash();
}
string server3Md5;
using (var resultFileContent = await server3.DownloadAsync("test.bin"))
{
server3Md5 = resultFileContent.GetMD5Hash();
}
Assert.Equal(server1Md5, server2Md5);
Assert.Equal(server2Md5, server3Md5);
Assert.Equal(1, server1.GetStatisticsAsync().Result.FileCount);
Assert.Equal(1, server2.GetStatisticsAsync().Result.FileCount);
Assert.Equal(1, server3.GetStatisticsAsync().Result.FileCount);
}
public async Task File_content_change_should_be_propagated()
{
var buffer = new byte[1024 * 1024 * 2]; // 2 MB
new Random().NextBytes(buffer);
var content = new MemoryStream(buffer);
var changedContent = new RandomlyModifiedStream(content, 0.02);
var server1 = NewClient(0);
var server2 = NewClient(1);
var server3 = NewClient(2);
content.Position = 0;
await server1.UploadAsync("test.bin", new RavenJObject { { "test", "value" } }, content);
Assert.Equal(1, server1.StatsAsync().Result.FileCount);
SyncTestUtils.TurnOnSynchronization(server1, server2);
Assert.Null(server1.Synchronization.SynchronizeDestinationsAsync().Result[0].Exception);
Assert.Equal(1, server2.StatsAsync().Result.FileCount);
SyncTestUtils.TurnOnSynchronization(server2, server3);
Assert.Null(server2.Synchronization.SynchronizeDestinationsAsync().Result[0].Exception);
Assert.Equal(1, server3.StatsAsync().Result.FileCount);
SyncTestUtils.TurnOffSynchronization(server1);
content.Position = 0;
await server1.UploadAsync("test.bin", changedContent);
SyncTestUtils.TurnOnSynchronization(server1, server2);
var secondServer1Synchronization = await server1.Synchronization.SynchronizeDestinationsAsync();
Assert.Null(secondServer1Synchronization[0].Exception);
Assert.Equal(SynchronizationType.ContentUpdate, secondServer1Synchronization[0].Reports.ToArray()[0].Type);
var secondServer2Synchronization = await server2.Synchronization.SynchronizeDestinationsAsync();
Assert.Null(secondServer2Synchronization[0].Exception);
Assert.Equal(SynchronizationType.ContentUpdate, secondServer2Synchronization[0].Reports.ToArray()[0].Type);
// On all servers should have the same content of the file
string server1Md5;
using (var resultFileContent = new MemoryStream())
{
server1.DownloadAsync("test.bin", resultFileContent).Wait();
resultFileContent.Position = 0;
server1Md5 = resultFileContent.GetMD5Hash();
}
string server2Md5;
using (var resultFileContent = new MemoryStream())
{
server2.DownloadAsync("test.bin", resultFileContent).Wait();
resultFileContent.Position = 0;
server2Md5 = resultFileContent.GetMD5Hash();
}
string server3Md5;
using (var resultFileContent = new MemoryStream())
{
server3.DownloadAsync("test.bin", resultFileContent).Wait();
resultFileContent.Position = 0;
server3Md5 = resultFileContent.GetMD5Hash();
}
Assert.Equal(server1Md5, server2Md5);
Assert.Equal(server2Md5, server3Md5);
Assert.Equal(1, server1.StatsAsync().Result.FileCount);
Assert.Equal(1, server2.StatsAsync().Result.FileCount);
Assert.Equal(1, server3.StatsAsync().Result.FileCount);
}