public async Task Can_synchronize_file_that_doesnt_have_any_signature_while_file_on_destination_has()
{
const int size1B = 1;
const int size5Mb = 1024 * 1024 * 5;
var source = NewAsyncClient(0);
var destination = NewAsyncClient(1);
var buffer = new byte[size1B]; // 1b file should have no signatures
new Random().NextBytes(buffer);
var sourceContent = new MemoryStream(buffer);
await source.UploadAsync("test.bin", sourceContent);
buffer = new byte[size5Mb]; // 5Mb file should have 2 signatures
new Random().NextBytes(buffer);
await destination.UploadAsync("test.bin", new MemoryStream(buffer));
var sourceSigCount = source.Synchronization.GetRdcManifestAsync("test.bin").Result.Signatures.Count;
var destinationSigCount = destination.Synchronization.GetRdcManifestAsync("test.bin").Result.Signatures.Count;
Assert.Equal(0, sourceSigCount); // ensure that file on source has no signature
Assert.True(destinationSigCount > 0, "File on destination should have any signature");
var result = SyncTestUtils.ResolveConflictAndSynchronize(source, destination, "test.bin");
Assert.Null(result.Exception);
Assert.Equal(size1B, result.BytesTransfered);
sourceContent.Position = 0;
Assert.Equal(sourceContent.GetMD5Hash(), destination.GetMetadataForAsync("test.bin").Result.Value <string>("Content-MD5"));
}
public async Task Should_just_rename_file_in_synchronization_process()
{
var content = new MemoryStream(new byte[] { 1, 2, 3, 4 });
var sourceClient = NewAsyncClient(0);
var destinationClient = NewAsyncClient(1);
await sourceClient.UploadAsync("test.bin", content, new RavenJObject { { "key", "value" } });
content.Position = 0;
await destinationClient.UploadAsync("test.bin", content, new RavenJObject { { "key", "value" } });
await sourceClient.RenameAsync("test.bin", "renamed.bin");
// we need to indicate old file name, otherwise content update would be performed because renamed file does not exist on dest
var report = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.bin");
Assert.Equal(SynchronizationType.Rename, report.Type);
var conflictItem = destinationClient.Configuration.GetKeyAsync <ConflictItem>(RavenFileNameHelper.ConflictConfigNameForFile("test.bin")).Result;
Assert.Null(conflictItem);
var testMetadata = await destinationClient.GetMetadataForAsync("test.bin");
var renamedMetadata = await destinationClient.GetMetadataForAsync("renamed.bin");
Assert.Null(testMetadata);
Assert.NotNull(renamedMetadata);
var result = await destinationClient.SearchOnDirectoryAsync("/");
Assert.Equal(1, result.FileCount);
Assert.Equal("renamed.bin", result.Files[0].Name);
}
public void Can_synchronize_file_with_less_number_of_signatures()
{
const int size5Mb = 1024 * 1024 * 5;
const int size1Mb = 1024 * 1024;
var source = NewAsyncClient(0);
var destination = NewAsyncClient(1);
var buffer = new byte[size1Mb]; // 1Mb file should have 1 signature
new Random().NextBytes(buffer);
var sourceContent = new MemoryStream(buffer);
source.UploadAsync("test.bin", sourceContent).Wait();
buffer = new byte[size5Mb]; // while 5Mb file has 2 signatures
new Random().NextBytes(buffer);
destination.UploadAsync("test.bin", new MemoryStream(buffer)).Wait();
var sourceSigCount = source.Synchronization.GetRdcManifestAsync("test.bin").Result.Signatures.Count;
var destinationSigCount = destination.Synchronization.GetRdcManifestAsync("test.bin").Result.Signatures.Count;
Assert.True(sourceSigCount > 0, "Source file should have one signature");
// ensure that file on source has less signatures than file on destination
Assert.True(sourceSigCount < destinationSigCount, "File on source should be smaller in order to have less signatures");
var result = SyncTestUtils.ResolveConflictAndSynchronize(source, destination, "test.bin");
Assert.Null(result.Exception);
Assert.Equal(size1Mb, result.BytesTransfered + result.BytesCopied);
sourceContent.Position = 0;
Assert.Equal(sourceContent.GetMD5Hash(), destination.GetMetadataForAsync("test.bin").Result.Value <string>("Content-MD5"));
}
public async Task Should_have_the_same_content(int size)
{
var sourceContent = SyncTestUtils.PrepareSourceStream(size);
sourceContent.Position = 0;
var destinationContent = new RandomlyModifiedStream(sourceContent, 0.01);
var destinationClient = NewAsyncClient(0);
var sourceClient = NewAsyncClient(1);
destinationClient.UploadAsync("test.txt", destinationContent, new RavenJObject()).Wait();
sourceContent.Position = 0;
sourceClient.UploadAsync("test.txt", sourceContent, new RavenJObject()).Wait();
var result = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.txt");
Assert.Equal(sourceContent.Length, result.BytesCopied + result.BytesTransfered);
string resultMd5;
using (var resultFileContent = await destinationClient.DownloadAsync("test.txt"))
{
resultMd5 = resultFileContent.GetMD5Hash();
}
sourceContent.Position = 0;
var sourceMd5 = sourceContent.GetMD5Hash();
Assert.Equal(sourceMd5, resultMd5);
}
public void Should_synchronize_just_metadata()
{
var content = new MemoryStream(new byte[] { 1, 2, 3, 4 });
var sourceClient = NewAsyncClient(0);
var destinationClient = NewAsyncClient(1);
sourceClient.UploadAsync("test.bin", content, new RavenJObject {
{ "difference", "metadata" }
}).Wait();
content.Position = 0;
destinationClient.UploadAsync("test.bin", content).Wait();
var report = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.bin");
var conflictItem = destinationClient.Configuration.GetKeyAsync <ConflictItem>(RavenFileNameHelper.ConflictConfigNameForFile("test.bin")).Result;
Assert.Null(conflictItem);
Assert.Equal(SynchronizationType.MetadataUpdate, report.Type);
var destinationMetadata = destinationClient.GetMetadataForAsync("test.bin").Result;
Assert.Equal("metadata", destinationMetadata.Value <string>("difference"));
}
public async Task Synchronize_file_with_appended_data(int size)
{
var differenceChunk = new MemoryStream();
var sw = new StreamWriter(differenceChunk);
sw.Write("Coconut is Stupid");
sw.Flush();
var sourceContent = new CombinedStream(SyncTestUtils.PrepareSourceStream(size), differenceChunk) {Position = 0};
var destinationContent = SyncTestUtils.PrepareSourceStream(size);
destinationContent.Position = 0;
var sourceClient = NewAsyncClient(0);
var destinationClient = NewAsyncClient(1);
await destinationClient.UploadAsync("test.txt", destinationContent);
sourceContent.Position = 0;
await sourceClient.UploadAsync("test.txt", sourceContent);
var result = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.txt");
Assert.Equal(sourceContent.Length, result.BytesCopied + result.BytesTransfered);
string resultMd5;
using (var resultFileContent = await destinationClient.DownloadAsync("test.txt"))
{
resultMd5 = resultFileContent.GetMD5Hash();
}
sourceContent.Position = 0;
var sourceMd5 = sourceContent.GetMD5Hash();
Assert.True(resultMd5 == sourceMd5);
}
public async Task Synchronization_of_already_synchronized_file_should_detect_that_no_work_is_needed(int size, int?seed)
{
Random r;
r = seed != null ? new Random(seed.Value) : new Random();
var bytes = new byte[size];
r.NextBytes(bytes);
var sourceContent = new MemoryStream(bytes);
var destinationContent = new RandomlyModifiedStream(new RandomStream(size, 1), 0.01, seed);
var destinationClient = NewAsyncClient(0);
var sourceClient = NewAsyncClient(1);
var srcMd5 = sourceContent.GetMD5Hash();
sourceContent.Position = 0;
var dstMd5 = (new RandomlyModifiedStream(new RandomStream(size, 1), 0.01, seed)).GetMD5Hash();
await destinationClient.UploadAsync("test.bin", destinationContent, new RavenJObject());
await sourceClient.UploadAsync("test.bin", sourceContent, new RavenJObject());
var firstSynchronization = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.bin");
Assert.Equal(sourceContent.Length, firstSynchronization.BytesCopied + firstSynchronization.BytesTransfered);
string resultMd5;
using (var resultFileContent = await destinationClient.DownloadAsync("test.bin"))
{
resultMd5 = resultFileContent.GetMD5Hash();
}
sourceContent.Position = 0;
var sourceMd5 = sourceContent.GetMD5Hash();
Assert.Equal(sourceMd5, resultMd5);
var secondSynchronization = sourceClient.Synchronization.StartAsync("test.bin", destinationClient).Result;
using (var resultFileContent = await destinationClient.DownloadAsync("test.bin"))
{
resultMd5 = resultFileContent.GetMD5Hash();
}
sourceContent.Position = 0;
sourceMd5 = sourceContent.GetMD5Hash();
Assert.Equal(sourceMd5, resultMd5);
Assert.Equal(0, secondSynchronization.NeedListLength);
Assert.Equal(0, secondSynchronization.BytesTransfered);
Assert.Equal(0, secondSynchronization.BytesCopied);
Assert.Equal("Destination server had this file in the past", secondSynchronization.Exception.Message);
}
public async Task Should_successfully_synchronize_if_last_synchronization_timeout_exceeded()
{
IAsyncFilesCommands destinationClient;
IAsyncFilesCommands sourceClient;
UploadFilesSynchronously(out sourceClient, out destinationClient);
await destinationClient.Configuration.SetKeyAsync(SynchronizationConstants.RavenSynchronizationLockTimeout, TimeSpan.FromSeconds(0));
Assert.DoesNotThrow(() => SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.bin"));
}
public async Task Should_refuse_to_synchronize_file_while_sync_configuration_exists()
{
IAsyncFilesCommands destinationClient;
IAsyncFilesCommands sourceClient;
UploadFilesSynchronously(out sourceClient, out destinationClient);
await destinationClient.Configuration.SetKeyAsync(RavenFileNameHelper.SyncLockNameForFile("test.bin"), SynchronizationConfig(DateTime.UtcNow));
var synchronizationReport = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.bin");
Assert.Equal(string.Format("File {0} is being synced", FileHeader.Canonize("test.bin")), synchronizationReport.Exception.Message);
}
public async Task Synchronize_file_with_different_beginning(int size)
{
var differenceChunk = new MemoryStream();
var sw = new StreamWriter(differenceChunk);
sw.Write("Coconut is Stupid");
sw.Flush();
var sourceContent = SyncTestUtils.PrepareSourceStream(size);
sourceContent.Position = 0;
var destinationContent = new CombinedStream(differenceChunk, sourceContent)
{
Position = 0
};
var sourceClient = NewAsyncClient(0);
var destinationClient = NewAsyncClient(1);
var sourceMetadata = new RavenJObject
{
{ "SomeTest-metadata", "some-value" }
};
var destinationMetadata = new RavenJObject
{
{ "SomeTest-metadata", "should-be-overwritten" }
};
await destinationClient.UploadAsync("test.txt", destinationContent, destinationMetadata);
sourceContent.Position = 0;
await sourceClient.UploadAsync("test.txt", sourceContent, sourceMetadata);
var result = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.txt");
Assert.Equal(sourceContent.Length, result.BytesCopied + result.BytesTransfered);
string resultMd5;
using (var resultFileContent = await destinationClient.DownloadAsync("test.txt"))
{
var metadata = await destinationClient.GetMetadataForAsync("test.txt");
Assert.Equal("some-value", metadata.Value <string>("SomeTest-Metadata"));
resultMd5 = resultFileContent.GetMD5Hash();
}
sourceContent.Position = 0;
var sourceMd5 = sourceContent.GetMD5Hash();
Assert.True(resultMd5 == sourceMd5);
}
public void Should_create_new_etag_for_replicated_file()
{
var destinationClient = NewAsyncClient(0);
var sourceClient = NewAsyncClient(1);
sourceClient.UploadAsync("test.bin", new RandomStream(10)).Wait();
destinationClient.UploadAsync("test.bin", new RandomStream(10)).Wait();
var destinationEtag = sourceClient.GetMetadataForAsync("test.bin").Result[Constants.MetadataEtagField];
SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.bin");
var result = destinationClient.GetMetadataForAsync("test.bin").Result[Constants.MetadataEtagField];
Assert.True(destinationEtag != result, "Etag should be updated");
}
public void Should_calculate_and_save_content_hash_after_synchronization()
{
var buffer = new byte[1024*1024*5 + 10];
new Random().NextBytes(buffer);
var sourceContent = new MemoryStream(buffer);
var sourceClient = NewAsyncClient(0);
sourceClient.UploadAsync("test.bin", sourceContent).Wait();
sourceContent.Position = 0;
var destinationClient = NewAsyncClient(1);
destinationClient.UploadAsync("test.bin", new RandomlyModifiedStream(sourceContent, 0.01)).Wait();
sourceContent.Position = 0;
SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.bin");
var resultFileMetadata = destinationClient.GetMetadataForAsync("test.bin").Result;
Assert.Contains("Content-MD5", resultFileMetadata.Keys);
Assert.Equal(sourceContent.GetMD5Hash(), resultFileMetadata.Value<string>("Content-MD5"));
}
public async Task Big_character_file_test(long size)
{
var sourceContent = new RandomCharacterStream(size);
var destinationContent = new RandomlyModifiedStream(new RandomCharacterStream(size), 0.01);
var destinationClient = NewAsyncClient(0);
var sourceClient = NewAsyncClient(1);
var sourceMetadata = new RavenJObject
{
{"SomeTest-metadata", "some-value"}
};
var destinationMetadata = new RavenJObject
{
{"SomeTest-metadata", "should-be-overwritten"}
};
await destinationClient.UploadAsync("test.bin", destinationContent, destinationMetadata);
await sourceClient.UploadAsync("test.bin", sourceContent, sourceMetadata);
SynchronizationReport result = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.bin");
Assert.Equal(sourceContent.Length, result.BytesCopied + result.BytesTransfered);
}
public async Task Can_get_synchronization_status()
{
var source = NewAsyncClient(0);
var destination = NewAsyncClient(1);
await source.UploadAsync("test.bin", new RandomStream(1024));
await destination.UploadAsync("test.bin", new RandomStream(1024));
var expected = SyncTestUtils.ResolveConflictAndSynchronize(source, destination, "test.bin");
var result = await destination.Synchronization.GetSynchronizationStatusForAsync("test.bin");
Assert.Equal(expected.BytesCopied, result.BytesCopied);
Assert.Equal(expected.BytesTransfered, result.BytesTransfered);
Assert.Equal(expected.Exception, result.Exception);
Assert.Equal(expected.FileETag, result.FileETag);
Assert.Equal(expected.FileName, result.FileName);
Assert.Equal(expected.NeedListLength, result.NeedListLength);
Assert.Equal(expected.Type, result.Type);
}
public void Should_get_all_finished_synchronizations()
{
var destinationClient = NewAsyncClient(0);
var sourceClient = NewAsyncClient(1);
var files = new[] { "test1.bin", "test2.bin", "test3.bin" };
// make sure that returns empty list if there are no finished synchronizations yet
var result = destinationClient.Synchronization.GetFinishedAsync().Result;
Assert.Equal(0, result.TotalCount);
foreach (var item in files)
{
var sourceContent = new MemoryStream();
var sw = new StreamWriter(sourceContent);
sw.Write("abc123");
sw.Flush();
sourceContent.Position = 0;
var destinationContent = new MemoryStream();
var sw2 = new StreamWriter(destinationContent);
sw2.Write("cba321");
sw2.Flush();
destinationContent.Position = 0;
Task.WaitAll(
destinationClient.UploadAsync(item, destinationContent),
sourceClient.UploadAsync(item, sourceContent));
SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, item);
}
result = destinationClient.Synchronization.GetFinishedAsync().Result;
Assert.Equal(files.Length, result.TotalCount);
}
public async Task Should_resolve_copied_files()
{
var source = NewAsyncClient(0);
var destination = NewAsyncClient(1);
await source.UploadAsync("test.bin", new RandomStream(1024));
await source.CopyAsync("test.bin", "test-copy.bin");
await destination.UploadAsync("test.bin", new RandomStream(1024));
await destination.CopyAsync("test.bin", "test-copy.bin");
var result = SyncTestUtils.ResolveConflictAndSynchronize(source, destination, "test-copy.bin");
Assert.Null(result.Exception);
var destinationStream = await destination.DownloadAsync("test-copy.bin");
var sourceStream = await destination.DownloadAsync("test-copy.bin");
Assert.Equal(sourceStream.GetMD5Hash(), destinationStream.GetMD5Hash());
}