public void Should_mark_file_to_be_resolved_using_current_strategy() { var differenceChunk = new MemoryStream(); var sw = new StreamWriter(differenceChunk); sw.Write("Coconut is Stupid"); sw.Flush(); var sourceContent = SyncTestUtils.PrepareSourceStream(10); sourceContent.Position = 0; var destinationContent = new CombinedStream(differenceChunk, sourceContent); var destinationClient = NewClient(0); var sourceClient = NewClient(1); var sourceMetadata = new RavenJObject { {"SomeTest-metadata", "some-value"} }; var destinationMetadata = new RavenJObject { {"SomeTest-metadata", "shouldnt-be-overwritten"} }; destinationClient.UploadAsync("test.txt", destinationMetadata, destinationContent).Wait(); sourceContent.Position = 0; sourceClient.UploadAsync("test.txt", sourceMetadata, sourceContent).Wait(); var shouldBeConflict = sourceClient.Synchronization.StartAsync("test.txt", destinationClient).Result; Assert.Equal("File test.txt is conflicted", shouldBeConflict.Exception.Message); destinationClient.Synchronization.ResolveConflictAsync("test.txt", ConflictResolutionStrategy.CurrentVersion).Wait(); var result = destinationClient.Synchronization.StartAsync("test.txt", sourceClient).Result; Assert.Equal(destinationContent.Length, result.BytesCopied + result.BytesTransfered); // check if conflict resolution has been properly set on the source string resultMd5; using (var resultFileContent = new MemoryStream()) { var metadata = sourceClient.DownloadAsync("test.txt", resultFileContent).Result; Assert.Equal("shouldnt-be-overwritten", metadata.Value<string>("SomeTest-Metadata")); resultFileContent.Position = 0; resultMd5 = resultFileContent.GetMD5Hash(); resultFileContent.Position = 0; } destinationContent.Position = 0; var destinationMd5 = destinationContent.GetMD5Hash(); sourceContent.Position = 0; Assert.True(resultMd5 == destinationMd5); }
public async Task CanWorkWithWinAuthEnabled() { var client = NewAsyncClient(enableAuthentication: true, credentials: new NetworkCredential(FactIfWindowsAuthenticationIsAvailable.Admin.UserName, FactIfWindowsAuthenticationIsAvailable.Admin.Password, FactIfWindowsAuthenticationIsAvailable.Admin.Domain)); var ms = new MemoryStream(new byte[1024 * 1024 * 10]); await client.UploadAsync("/dir/ms.bin", ms); ms.Position = 0; await client.UploadAsync("/dir/ms.bin", ms); var result = new MemoryStream(); (await client.DownloadAsync("/dir/ms.bin")).CopyTo(result); ms.Position = 0; result.Position = 0; Assert.Equal(ms.GetMD5Hash(), result.GetMD5Hash()); await client.RenameAsync("/dir/ms.bin", "/dir/sm.bin"); var searchResults = await client.SearchOnDirectoryAsync("/dir"); Assert.Equal(1, searchResults.FileCount); var metadata = await client.GetMetadataForAsync("/dir/sm.bin"); Assert.NotNull(metadata); var folders = await client.GetDirectoriesAsync(); Assert.Equal(1, folders.Length); var searchFields = await client.GetSearchFieldsAsync(); Assert.True(searchFields.Length > 0); var guid = await client.GetServerIdAsync(); Assert.NotEqual(Guid.Empty, guid); await client.UpdateMetadataAsync("/dir/sm.bin", new RavenJObject() { { "Meta", "Data" } }); var results = await client.SearchAsync("Meta:Data"); Assert.Equal(1, results.FileCount); var stats = await client.GetStatisticsAsync(); Assert.Equal(1, stats.FileCount); }
public async Task CanWorkWithWinAuthEnabled() { var client = NewClient(enableAuthentication: true, credentials: new NetworkCredential(username, password, domain)); var ms = new MemoryStream(new byte[]{1, 2, 4}); await client.UploadAsync("/dir/ms.bin", ms); var result = new MemoryStream(); await client.DownloadAsync("/dir/ms.bin", result); ms.Position = 0; result.Position = 0; Assert.Equal(ms.GetMD5Hash(), result.GetMD5Hash()); await client.RenameAsync("/dir/ms.bin", "/dir/sm.bin"); var searchResults = await client.GetFilesAsync("/dir"); Assert.Equal(1, searchResults.FileCount); var metadata = await client.GetMetadataForAsync("/dir/sm.bin"); Assert.NotNull(metadata); var folders = await client.GetFoldersAsync(); Assert.Equal(1, folders.Length); var searchFields = await client.GetSearchFieldsAsync(); Assert.True(searchFields.Length > 0); var guid = await client.GetServerId(); Assert.NotEqual(Guid.Empty, guid); await client.UpdateMetadataAsync("/dir/sm.bin", new RavenJObject() { { "Meta", "Data" } }); var results = await client.SearchAsync("Meta:Data"); Assert.Equal(1, results.FileCount); var stats = await client.StatsAsync(); Assert.Equal(1, stats.FileCount); }
public async void Synchronize_file_with_different_beginning(int size) { var differenceChunk = new MemoryStream(); var sw = new StreamWriter(differenceChunk); sw.Write("Coconut is Stupid"); sw.Flush(); var sourceContent = SyncTestUtils.PrepareSourceStream(size); sourceContent.Position = 0; var destinationContent = new CombinedStream(differenceChunk, sourceContent) {Position = 0}; var sourceClient = NewClient(0); var destinationClient = NewClient(1); var sourceMetadata = new RavenJObject { {"SomeTest-metadata", "some-value"} }; var destinationMetadata = new RavenJObject { {"SomeTest-metadata", "should-be-overwritten"} }; await destinationClient.UploadAsync("test.txt", destinationMetadata, destinationContent); sourceContent.Position = 0; await sourceClient.UploadAsync("test.txt", sourceMetadata, sourceContent); var result = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.txt"); Assert.Equal(sourceContent.Length, result.BytesCopied + result.BytesTransfered); string resultMd5; using (var resultFileContent = new MemoryStream()) { var metadata = destinationClient.DownloadAsync("test.txt", resultFileContent).Result; // REVIEW: (Oren) The xxx-yyy-zzz headers are being transformed to: Xxx-Yyy-Zzz by the underlying implementation of HTTP Client. Is that OK? The old test was able to handle it "case insensitively". Assert.Equal("some-value", metadata.Value<string>("SomeTest-Metadata")); resultFileContent.Position = 0; resultMd5 = resultFileContent.GetMD5Hash(); resultFileContent.Position = 0; } sourceContent.Position = 0; var sourceMd5 = sourceContent.GetMD5Hash(); Assert.True(resultMd5 == sourceMd5); }
public void Synchronize_file_with_different_beginning(int size) { var differenceChunk = new MemoryStream(); var sw = new StreamWriter(differenceChunk); sw.Write("Coconut is Stupid"); sw.Flush(); var sourceContent = SyncTestUtils.PrepareSourceStream(size); sourceContent.Position = 0; var destinationContent = new CombinedStream(differenceChunk, sourceContent) {Position = 0}; var sourceClient = NewClient(0); var destinationClient = NewClient(1); var sourceMetadata = new NameValueCollection { {"SomeTest-metadata", "some-value"} }; var destinationMetadata = new NameValueCollection { {"SomeTest-metadata", "should-be-overwritten"} }; destinationClient.UploadAsync("test.txt", destinationMetadata, destinationContent).Wait(); sourceContent.Position = 0; sourceClient.UploadAsync("test.txt", sourceMetadata, sourceContent).Wait(); var result = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.txt"); Assert.Equal(sourceContent.Length, result.BytesCopied + result.BytesTransfered); string resultMd5; using (var resultFileContent = new MemoryStream()) { var metadata = destinationClient.DownloadAsync("test.txt", resultFileContent).Result; Assert.Equal("some-value", metadata["SomeTest-metadata"]); resultFileContent.Position = 0; resultMd5 = resultFileContent.GetMD5Hash(); resultFileContent.Position = 0; } sourceContent.Position = 0; var sourceMd5 = sourceContent.GetMD5Hash(); Assert.True(resultMd5 == sourceMd5); }
public async Task uploading_file_multiple_times_must_not_throw_key_duplicate_exception_on_esent_and_concurrency_exception_on_voron(string storage) { var r = new Random(1); var bytes = new byte[1024]; r.NextBytes(bytes); var ms = new MemoryStream(bytes); var expectedHash = ms.GetMD5Hash(); var client = NewAsyncClient(requestedStorage: storage); for (int i = 0; i < 500; i++) { ms.Position = 0; await client.UploadAsync("abc.bin", ms); } var stream = await client.DownloadAsync("abc.bin"); Assert.Equal(expectedHash, stream.GetMD5Hash()); }
public async Task Should_transfer_entire_file_even_if_rename_operation_was_performed() { var source = NewClient(0); var destination = NewClient(1); var fileContent = new MemoryStream(new byte[] {1, 2, 3}); await source.UploadAsync("test.bin", fileContent); await source.RenameAsync("test.bin", "renamed.bin"); SyncTestUtils.TurnOnSynchronization(source, destination); var destinationSyncResults = await source.Synchronization.SynchronizeDestinationsAsync(); Assert.Equal(1, destinationSyncResults.Length); var reports = destinationSyncResults[0].Reports.ToArray(); Assert.Null(reports[0].Exception); Assert.Equal(SynchronizationType.ContentUpdate, reports[0].Type); Assert.Equal("renamed.bin", reports[0].FileName); fileContent.Position = 0; Assert.Equal(fileContent.GetMD5Hash(), destination.GetMetadataForAsync("renamed.bin").Result["Content-MD5"]); }
public void ShouldWork() { var client = NewClient(); var tasks = new List<Task>(10); // upload 10 files with the same content but different names concurrently Assert.DoesNotThrow( () => Parallel.For(0, 10, x => tasks.Add(client.UploadAsync("test" + x, new MemoryStream(new byte[] {1, 2, 3, 4, 5}))))); Task.WaitAll(tasks.ToArray()); var hash = new MemoryStream(new byte[] {1, 2, 3, 4, 5}).GetMD5Hash(); for (var i = 0; i < 10; i++) { var uploadedContent = new MemoryStream(); client.DownloadAsync("test" + i, uploadedContent).Wait(); uploadedContent.Position = 0; Assert.Equal(hash, uploadedContent.GetMD5Hash()); } }
public void Should_synchronize_to_all_destinations() { StartServerInstance(AddtitionalServerInstancePortNumber); var sourceContent = SyncTestUtils.PrepareSourceStream(10000); sourceContent.Position = 0; var sourceClient = NewClient(0); var destination1Client = NewClient(1); var destination2Client = new RavenFileSystemClient(ServerAddress(AddtitionalServerInstancePortNumber)); var destination1Content = new RandomlyModifiedStream(sourceContent, 0.01); sourceContent.Position = 0; var destination2Content = new RandomlyModifiedStream(sourceContent, 0.01); sourceContent.Position = 0; destination1Client.UploadAsync("test.bin", destination1Content).Wait(); destination2Client.UploadAsync("test.bin", destination2Content).Wait(); sourceContent.Position = 0; sourceClient.UploadAsync("test.bin", sourceContent).Wait(); sourceContent.Position = 0; sourceClient.Config.SetConfig(SynchronizationConstants.RavenSynchronizationDestinations, new NameValueCollection { { "url", destination1Client .ServerUrl }, { "url", destination2Client .ServerUrl } }).Wait(); var destinationSyncResults = sourceClient.Synchronization.SynchronizeDestinationsAsync().Result; // we expect conflicts after first attempt of synchronization Assert.Equal(2, destinationSyncResults.Length); Assert.Equal("File test.bin is conflicted", destinationSyncResults[0].Reports.ToArray()[0].Exception.Message); Assert.Equal("File test.bin is conflicted", destinationSyncResults[1].Reports.ToArray()[0].Exception.Message); destination1Client.Synchronization.ResolveConflictAsync("test.bin", ConflictResolutionStrategy.RemoteVersion).Wait(); destination2Client.Synchronization.ResolveConflictAsync("test.bin", ConflictResolutionStrategy.RemoteVersion).Wait(); destinationSyncResults = sourceClient.Synchronization.SynchronizeDestinationsAsync().Result; // check if reports match Assert.Equal(2, destinationSyncResults.Length); var result1 = destinationSyncResults[0].Reports.ToArray()[0]; Assert.Equal(sourceContent.Length, result1.BytesCopied + result1.BytesTransfered); var result2 = destinationSyncResults[1].Reports.ToArray()[0]; Assert.Equal(sourceContent.Length, result2.BytesCopied + result2.BytesTransfered); // check content of files string destination1Md5; using (var resultFileContent = new MemoryStream()) { destination1Client.DownloadAsync("test.bin", resultFileContent).Wait(); resultFileContent.Position = 0; destination1Md5 = resultFileContent.GetMD5Hash(); } string destination2Md5; using (var resultFileContent = new MemoryStream()) { destination2Client.DownloadAsync("test.bin", resultFileContent).Wait(); resultFileContent.Position = 0; destination2Md5 = resultFileContent.GetMD5Hash(); } sourceContent.Position = 0; var sourceMd5 = sourceContent.GetMD5Hash(); Assert.Equal(sourceMd5, destination1Md5); Assert.Equal(sourceMd5, destination2Md5); Assert.Equal(destination1Md5, destination2Md5); }
public async Task After_file_delete_next_synchronization_should_override_tombsone() { var source = NewAsyncClient(0); var destination = NewAsyncClient(1); var sourceContent = new MemoryStream(new byte[] {5, 10, 15}) {Position = 0}; await source.UploadAsync("test.bin", sourceContent); var report = await source.Synchronization.StartAsync("test.bin", destination); Assert.Null(report.Exception); await destination.DeleteAsync("test.bin"); report = await source.Synchronization.StartAsync("test.bin", destination); Assert.Null(report.Exception); var destContent = await destination.DownloadAsync("test.bin"); var destMetadata = await destination.GetMetadataForAsync("test.bin"); Assert.True(destMetadata[SynchronizationConstants.RavenDeleteMarker] == null, "Metadata should not containt Raven-Delete-Marker"); sourceContent.Position = 0; Assert.Equal(sourceContent.GetMD5Hash(), destContent.GetMD5Hash()); }
public async Task Can_synchronize_file_that_doesnt_have_any_signature_while_file_on_destination_has() { const int size1B = 1; const int size5Mb = 1024*1024*5; var source = NewAsyncClient(0); var destination = NewAsyncClient(1); var buffer = new byte[size1B]; // 1b file should have no signatures new Random().NextBytes(buffer); var sourceContent = new MemoryStream(buffer); await source.UploadAsync("test.bin", sourceContent); buffer = new byte[size5Mb]; // 5Mb file should have 2 signatures new Random().NextBytes(buffer); await destination.UploadAsync("test.bin", new MemoryStream(buffer)); var sourceSigCount = source.Synchronization.GetRdcManifestAsync("test.bin").Result.Signatures.Count; var destinationSigCount = destination.Synchronization.GetRdcManifestAsync("test.bin").Result.Signatures.Count; Assert.Equal(0, sourceSigCount); // ensure that file on source has no signature Assert.True(destinationSigCount > 0, "File on destination should have any signature"); var result = SyncTestUtils.ResolveConflictAndSynchronize(source, destination, "test.bin"); Assert.Null(result.Exception); Assert.Equal(size1B, result.BytesTransfered); sourceContent.Position = 0; Assert.Equal(sourceContent.GetMD5Hash(), destination.GetMetadataForAsync("test.bin").Result.Value<string>("Content-MD5")); }
public void Can_synchronize_file_with_less_number_of_signatures() { const int size5Mb = 1024*1024*5; const int size1Mb = 1024*1024; var source = NewAsyncClient(0); var destination = NewAsyncClient(1); var buffer = new byte[size1Mb]; // 1Mb file should have 1 signature new Random().NextBytes(buffer); var sourceContent = new MemoryStream(buffer); source.UploadAsync("test.bin", sourceContent).Wait(); buffer = new byte[size5Mb]; // while 5Mb file has 2 signatures new Random().NextBytes(buffer); destination.UploadAsync("test.bin", new MemoryStream(buffer)).Wait(); var sourceSigCount = source.Synchronization.GetRdcManifestAsync("test.bin").Result.Signatures.Count; var destinationSigCount = destination.Synchronization.GetRdcManifestAsync("test.bin").Result.Signatures.Count; Assert.True(sourceSigCount > 0, "Source file should have one signature"); // ensure that file on source has less signatures than file on destination Assert.True(sourceSigCount < destinationSigCount, "File on source should be smaller in order to have less signatures"); var result = SyncTestUtils.ResolveConflictAndSynchronize(source, destination, "test.bin"); Assert.Null(result.Exception); Assert.Equal(size1Mb, result.BytesTransfered + result.BytesCopied); sourceContent.Position = 0; Assert.Equal(sourceContent.GetMD5Hash(), destination.GetMetadataForAsync("test.bin").Result.Value<string>("Content-MD5")); }
public void Should_not_change_content_hash_after_metadata_upload() { var buffer = new byte[1024]; new Random().NextBytes(buffer); var sourceContent = new MemoryStream(buffer); var sourceClient = NewAsyncClient(0); sourceClient.UploadAsync("test.bin", sourceContent).Wait(); sourceClient.UpdateMetadataAsync("test.bin", new RavenJObject { { "someKey", "someValue" } }).Wait(); sourceContent.Position = 0; var resultFileMetadata = sourceClient.GetMetadataForAsync("test.bin").Result; Assert.Contains("Content-MD5", resultFileMetadata.Keys); Assert.Equal(sourceContent.GetMD5Hash(), resultFileMetadata.Value<string>("Content-MD5")); }
public void Should_calculate_and_save_content_hash_after_synchronization() { var buffer = new byte[1024*1024*5 + 10]; new Random().NextBytes(buffer); var sourceContent = new MemoryStream(buffer); var sourceClient = NewAsyncClient(0); sourceClient.UploadAsync("test.bin", sourceContent).Wait(); sourceContent.Position = 0; var destinationClient = NewAsyncClient(1); destinationClient.UploadAsync("test.bin", new RandomlyModifiedStream(sourceContent, 0.01)).Wait(); sourceContent.Position = 0; SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.bin"); var resultFileMetadata = destinationClient.GetMetadataForAsync("test.bin").Result; Assert.Contains("Content-MD5", resultFileMetadata.Keys); Assert.Equal(sourceContent.GetMD5Hash(), resultFileMetadata.Value<string>("Content-MD5")); }
public async Task Synchronization_of_already_synchronized_file_should_detect_that_no_work_is_needed(int size, int? seed) { Random r; r = seed != null ? new Random(seed.Value) : new Random(); var bytes = new byte[size]; r.NextBytes(bytes); var sourceContent = new MemoryStream(bytes); var destinationContent = new RandomlyModifiedStream(new RandomStream(size, 1), 0.01, seed); var destinationClient = NewAsyncClient(0); var sourceClient = NewAsyncClient(1); var srcMd5 = sourceContent.GetMD5Hash(); sourceContent.Position = 0; var dstMd5 = (new RandomlyModifiedStream(new RandomStream(size, 1), 0.01, seed)).GetMD5Hash(); await destinationClient.UploadAsync("test.bin", destinationContent, new RavenJObject()); await sourceClient.UploadAsync("test.bin", sourceContent, new RavenJObject()); var firstSynchronization = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.bin"); Assert.Equal(sourceContent.Length, firstSynchronization.BytesCopied + firstSynchronization.BytesTransfered); string resultMd5; using (var resultFileContent = await destinationClient.DownloadAsync("test.bin")) { resultMd5 = resultFileContent.GetMD5Hash(); } sourceContent.Position = 0; var sourceMd5 = sourceContent.GetMD5Hash(); Assert.Equal(sourceMd5, resultMd5); var secondSynchronization = sourceClient.Synchronization.StartAsync("test.bin", destinationClient).Result; using (var resultFileContent = await destinationClient.DownloadAsync("test.bin")) { resultMd5 = resultFileContent.GetMD5Hash(); } sourceContent.Position = 0; sourceMd5 = sourceContent.GetMD5Hash(); Assert.Equal(sourceMd5, resultMd5); Assert.Equal(0, secondSynchronization.NeedListLength); Assert.Equal(0, secondSynchronization.BytesTransfered); Assert.Equal(0, secondSynchronization.BytesCopied); Assert.Equal("Destination server had this file in the past", secondSynchronization.Exception.Message); }
public void Should_reuse_second_page_if_only_first_one_changed() { string filename = FileHeader.Canonize("test"); var file = SyncTestUtils.PreparePagesStream(2); file.Position = 0; var sourceContent = new MemoryStream(); file.CopyTo(sourceContent); sourceContent.Position = 0; sourceContent.Write(new byte[] {0, 0, 0, 0}, 0, 4); // change content of the 1st page var destinationContent = file; sourceContent.Position = 0; source.UploadAsync(filename, sourceContent).Wait(); destinationContent.Position = 0; destination.UploadAsync(filename, destinationContent).Wait(); var contentUpdate = new ContentUpdateWorkItem(filename, "http://localhost:12345", sourceRfs.Storage, sourceRfs.SigGenerator); sourceContent.Position = 0; // force to upload entire file, we just want to check which pages will be reused contentUpdate.UploadToAsync(destination.Synchronization).Wait(); destination.Synchronization.ResolveConflictAsync(filename, ConflictResolutionStrategy.RemoteVersion).Wait(); contentUpdate.UploadToAsync(destination.Synchronization).Wait(); FileAndPagesInformation fileAndPages = null; destinationRfs.Storage.Batch(accessor => fileAndPages = accessor.GetFile(filename, 0, 256)); Assert.Equal(2, fileAndPages.Pages.Count); Assert.Equal(3, fileAndPages.Pages[0].Id); // new page -> id == 3 Assert.Equal(2, fileAndPages.Pages[1].Id); // reused page -> id still == 2 sourceContent.Position = 0; Assert.Equal(sourceContent.GetMD5Hash(), destination.GetMetadataForAsync(filename).Result["Content-MD5"]); }
public async Task Should_reuse_pages_where_nothing_has_changed() { string filename = FileHeader.Canonize("test"); var file = SyncTestUtils.PreparePagesStream(3); file.Position = 0; var sourceContent = new MemoryStream(); file.CopyTo(sourceContent); sourceContent.Position = StorageConstants.MaxPageSize + 1; sourceContent.Write(new byte[] {0, 0, 0, 0}, 0, 4); // change content of the 2nd page var destinationContent = file; sourceContent.Position = 0; await source.UploadAsync(filename, sourceContent); destinationContent.Position = 0; await destination.UploadAsync(filename, destinationContent); var contentUpdate = new ContentUpdateWorkItem(filename, "http://localhost:12345", sourceRfs.Storage, sourceRfs.SigGenerator); sourceContent.Position = 0; // force to upload entire file, we just want to check which pages will be reused await contentUpdate.UploadToAsync(destination.Synchronization); await destination.Synchronization.ResolveConflictAsync(filename, ConflictResolutionStrategy.RemoteVersion); await contentUpdate.UploadToAsync(destination.Synchronization); FileAndPagesInformation fileAndPages = null; destinationRfs.Storage.Batch(accessor => fileAndPages = accessor.GetFile(filename, 0, 256)); Assert.Equal(3, fileAndPages.Pages.Count); Assert.Equal(1, fileAndPages.Pages[0].Id); // reused page Assert.Equal(4, fileAndPages.Pages[1].Id); // new page -> id == 4 Assert.Equal(3, fileAndPages.Pages[2].Id); // reused page sourceContent.Position = 0; var metadata = await destination.GetMetadataForAsync(filename); Assert.Equal(sourceContent.GetMD5Hash(), metadata.Value<string>("Content-MD5")); }
public async Task File_content_change_should_be_propagated() { var buffer = new byte[1024*1024*2]; // 2 MB new Random().NextBytes(buffer); var content = new MemoryStream(buffer); var changedContent = new RandomlyModifiedStream(content, 0.02); var server1 = NewClient(0); var server2 = NewClient(1); var server3 = NewClient(2); content.Position = 0; await server1.UploadAsync("test.bin", new RavenJObject { { "test", "value" } }, content); Assert.Equal(1, server1.StatsAsync().Result.FileCount); SyncTestUtils.TurnOnSynchronization(server1, server2); Assert.Null(server1.Synchronization.SynchronizeDestinationsAsync().Result[0].Exception); Assert.Equal(1, server2.StatsAsync().Result.FileCount); SyncTestUtils.TurnOnSynchronization(server2, server3); Assert.Null(server2.Synchronization.SynchronizeDestinationsAsync().Result[0].Exception); Assert.Equal(1, server3.StatsAsync().Result.FileCount); SyncTestUtils.TurnOffSynchronization(server1); content.Position = 0; await server1.UploadAsync("test.bin", changedContent); SyncTestUtils.TurnOnSynchronization(server1, server2); var secondServer1Synchronization = await server1.Synchronization.SynchronizeDestinationsAsync(); Assert.Null(secondServer1Synchronization[0].Exception); Assert.Equal(SynchronizationType.ContentUpdate, secondServer1Synchronization[0].Reports.ToArray()[0].Type); var secondServer2Synchronization = await server2.Synchronization.SynchronizeDestinationsAsync(); Assert.Null(secondServer2Synchronization[0].Exception); Assert.Equal(SynchronizationType.ContentUpdate, secondServer2Synchronization[0].Reports.ToArray()[0].Type); // On all servers should have the same content of the file string server1Md5; using (var resultFileContent = new MemoryStream()) { server1.DownloadAsync("test.bin", resultFileContent).Wait(); resultFileContent.Position = 0; server1Md5 = resultFileContent.GetMD5Hash(); } string server2Md5; using (var resultFileContent = new MemoryStream()) { server2.DownloadAsync("test.bin", resultFileContent).Wait(); resultFileContent.Position = 0; server2Md5 = resultFileContent.GetMD5Hash(); } string server3Md5; using (var resultFileContent = new MemoryStream()) { server3.DownloadAsync("test.bin", resultFileContent).Wait(); resultFileContent.Position = 0; server3Md5 = resultFileContent.GetMD5Hash(); } Assert.Equal(server1Md5, server2Md5); Assert.Equal(server2Md5, server3Md5); Assert.Equal(1, server1.StatsAsync().Result.FileCount); Assert.Equal(1, server2.StatsAsync().Result.FileCount); Assert.Equal(1, server3.StatsAsync().Result.FileCount); }
public void Should_reuse_pages_where_nothing_has_changed() { var file = SyncTestUtils.PreparePagesStream(3); file.Position = 0; var sourceContent = new MemoryStream(); file.CopyTo(sourceContent); sourceContent.Position = StorageConstants.MaxPageSize + 1; sourceContent.Write(new byte[] {0, 0, 0, 0}, 0, 4); // change content of the 2nd page var destinationContent = file; sourceContent.Position = 0; source.UploadAsync("test", sourceContent).Wait(); destinationContent.Position = 0; destination.UploadAsync("test", destinationContent).Wait(); var contentUpdate = new ContentUpdateWorkItem("test", "http://localhost:12345", sourceRfs.Storage, sourceRfs.SigGenerator); sourceContent.Position = 0; // force to upload entire file, we just want to check which pages will be reused contentUpdate.UploadToAsync(destination.ServerUrl).Wait(); destination.Synchronization.ResolveConflictAsync("test", ConflictResolutionStrategy.RemoteVersion).Wait(); contentUpdate.UploadToAsync(destination.ServerUrl).Wait(); FileAndPages fileAndPages = null; destinationRfs.Storage.Batch(accessor => fileAndPages = accessor.GetFile("test", 0, 256)); Assert.Equal(3, fileAndPages.Pages.Count); Assert.Equal(1, fileAndPages.Pages[0].Id); // reused page Assert.Equal(4, fileAndPages.Pages[1].Id); // new page -> id == 4 Assert.Equal(3, fileAndPages.Pages[2].Id); // reused page sourceContent.Position = 0; Assert.Equal(sourceContent.GetMD5Hash(), destination.GetMetadataForAsync("test").Result["Content-MD5"]); }
public void Should_synchronize_to_all_destinations() { var sourceContent = SyncTestUtils.PrepareSourceStream(10000); sourceContent.Position = 0; var sourceClient = NewClient(0); var destination1Client = NewClient(1); var destination2Client = NewClient(2); var destination1Content = new RandomlyModifiedStream(sourceContent, 0.01); sourceContent.Position = 0; var destination2Content = new RandomlyModifiedStream(sourceContent, 0.01); sourceContent.Position = 0; destination1Client.UploadAsync("test.bin", destination1Content).Wait(); destination2Client.UploadAsync("test.bin", destination2Content).Wait(); sourceContent.Position = 0; sourceClient.UploadAsync("test.bin", sourceContent).Wait(); sourceContent.Position = 0; sourceClient.Config.SetDestinationsConfig(destination1Client.ToSynchronizationDestination(), destination2Client.ToSynchronizationDestination()).Wait(); var destinationSyncResults = sourceClient.Synchronization.SynchronizeDestinationsAsync().Result; // we expect conflicts after first attempt of synchronization Assert.Equal(2, destinationSyncResults.Length); Assert.Equal("File test.bin is conflicted", destinationSyncResults[0].Reports.ToArray()[0].Exception.Message); Assert.Equal("File test.bin is conflicted", destinationSyncResults[1].Reports.ToArray()[0].Exception.Message); destination1Client.Synchronization.ResolveConflictAsync("test.bin", ConflictResolutionStrategy.RemoteVersion).Wait(); destination2Client.Synchronization.ResolveConflictAsync("test.bin", ConflictResolutionStrategy.RemoteVersion).Wait(); destinationSyncResults = sourceClient.Synchronization.SynchronizeDestinationsAsync().Result; var conflictItem = destination1Client.Config.GetConfig<ConflictItem>(RavenFileNameHelper.ConflictConfigNameForFile("test.bin")).Result; Assert.Null(conflictItem); conflictItem = destination2Client.Config.GetConfig<ConflictItem>(RavenFileNameHelper.ConflictConfigNameForFile("test.bin")).Result; Assert.Null(conflictItem); // check if reports match Assert.Equal(2, destinationSyncResults.Length); var result1 = destinationSyncResults[0].Reports.ToArray()[0]; Assert.Equal(sourceContent.Length, result1.BytesCopied + result1.BytesTransfered); Assert.Equal(SynchronizationType.ContentUpdate, result1.Type); var result2 = destinationSyncResults[1].Reports.ToArray()[0]; Assert.Equal(sourceContent.Length, result2.BytesCopied + result2.BytesTransfered); Assert.Equal(SynchronizationType.ContentUpdate, result2.Type); // check content of files string destination1Md5; using (var resultFileContent = new MemoryStream()) { destination1Client.DownloadAsync("test.bin", resultFileContent).Wait(); resultFileContent.Position = 0; destination1Md5 = resultFileContent.GetMD5Hash(); } string destination2Md5; using (var resultFileContent = new MemoryStream()) { destination2Client.DownloadAsync("test.bin", resultFileContent).Wait(); resultFileContent.Position = 0; destination2Md5 = resultFileContent.GetMD5Hash(); } sourceContent.Position = 0; var sourceMd5 = sourceContent.GetMD5Hash(); Assert.Equal(sourceMd5, destination1Md5); Assert.Equal(sourceMd5, destination2Md5); Assert.Equal(destination1Md5, destination2Md5); }
public void Synchronize_file_with_appended_data(int size) { var differenceChunk = new MemoryStream(); var sw = new StreamWriter(differenceChunk); sw.Write("Coconut is Stupid"); sw.Flush(); var sourceContent = new CombinedStream(SyncTestUtils.PrepareSourceStream(size), differenceChunk) {Position = 0}; var destinationContent = SyncTestUtils.PrepareSourceStream(size); destinationContent.Position = 0; var sourceClient = NewClient(0); var destinationClient = NewClient(1); destinationClient.UploadAsync("test.txt", destinationContent).Wait(); sourceContent.Position = 0; sourceClient.UploadAsync("test.txt", sourceContent).Wait(); var result = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.txt"); Assert.Equal(sourceContent.Length, result.BytesCopied + result.BytesTransfered); string resultMd5; using (var resultFileContent = new MemoryStream()) { destinationClient.DownloadAsync("test.txt", resultFileContent).Wait(); resultFileContent.Position = 0; resultMd5 = resultFileContent.GetMD5Hash(); resultFileContent.Position = 0; } sourceContent.Position = 0; var sourceMd5 = sourceContent.GetMD5Hash(); Assert.True(resultMd5 == sourceMd5); }
public void Should_calculate_and_save_content_hash_after_upload() { var buffer = new byte[1024]; new Random().NextBytes(buffer); var sourceContent = new MemoryStream(buffer); var sourceClient = NewClient(0); sourceClient.UploadAsync("test.bin", sourceContent).Wait(); sourceContent.Position = 0; var resultFileMetadata = sourceClient.GetMetadataForAsync("test.bin").Result; Assert.Contains("Content-MD5", resultFileMetadata.AllKeys); Assert.Equal(sourceContent.GetMD5Hash(), resultFileMetadata["Content-MD5"]); }
public void Should_have_the_same_content(int size) { var sourceContent = SyncTestUtils.PrepareSourceStream(size); sourceContent.Position = 0; var destinationContent = new RandomlyModifiedStream(sourceContent, 0.01); var destinationClient = NewClient(0); var sourceClient = NewClient(1); destinationClient.UploadAsync("test.txt", new NameValueCollection(), destinationContent).Wait(); sourceContent.Position = 0; sourceClient.UploadAsync("test.txt", new NameValueCollection(), sourceContent).Wait(); var result = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.txt"); Assert.Equal(sourceContent.Length, result.BytesCopied + result.BytesTransfered); string resultMd5; using (var resultFileContent = new MemoryStream()) { destinationClient.DownloadAsync("test.txt", resultFileContent).Wait(); resultFileContent.Position = 0; resultMd5 = resultFileContent.GetMD5Hash(); } sourceContent.Position = 0; var sourceMd5 = sourceContent.GetMD5Hash(); Assert.Equal(sourceMd5, resultMd5); }