public override async Task<SynchronizationReport> PerformAsync(string destination)
{
AssertLocalFileExistsAndIsNotConflicted(FileMetadata);
var destinationRavenFileSystemClient = new RavenFileSystemClient(destination);
var destinationMetadata = await destinationRavenFileSystemClient.GetMetadataForAsync(FileName);
if (destinationMetadata == null)
{
// if file doesn't exist on destination server - upload it there
return await UploadToAsync(destination);
}
var destinationServerRdcStats = await destinationRavenFileSystemClient.Synchronization.GetRdcStatsAsync();
if (!IsRemoteRdcCompatible(destinationServerRdcStats))
throw new SynchronizationException("Incompatible RDC version detected on destination server");
var conflict = CheckConflictWithDestination(FileMetadata, destinationMetadata, ServerInfo.Url);
if (conflict != null)
return await ApplyConflictOnDestinationAsync(conflict, destination, ServerInfo.Url, log);
using (var localSignatureRepository = new StorageSignatureRepository(Storage, FileName))
using (var remoteSignatureCache = new VolatileSignatureRepository(FileName))
{
var localRdcManager = new LocalRdcManager(localSignatureRepository, Storage, sigGenerator);
var destinationRdcManager = new RemoteRdcManager(destinationRavenFileSystemClient, localSignatureRepository,
remoteSignatureCache);
log.Debug("Starting to retrieve signatures of a local file '{0}'.", FileName);
Cts.Token.ThrowIfCancellationRequested();
// first we need to create a local file signatures before we synchronize with remote ones
var localSignatureManifest = await localRdcManager.GetSignatureManifestAsync(FileDataInfo);
log.Debug("Number of a local file '{0}' signatures was {1}.", FileName, localSignatureManifest.Signatures.Count);
if (localSignatureManifest.Signatures.Count > 0)
{
var destinationSignatureManifest = await destinationRdcManager.SynchronizeSignaturesAsync(FileDataInfo, Cts.Token);
if (destinationSignatureManifest.Signatures.Count > 0)
{
return
await
SynchronizeTo(destination, localSignatureRepository, remoteSignatureCache, localSignatureManifest,
destinationSignatureManifest);
}
}
return await UploadToAsync(destination);
}
}
public void Should_reuse_pages_when_data_appended(int numberOfPages)
{
var file = SyncTestUtils.PreparePagesStream(numberOfPages);
var sourceContent = new CombinedStream(file, SyncTestUtils.PreparePagesStream(numberOfPages));
// add new pages at the end
var destinationContent = file;
sourceContent.Position = 0;
source.UploadAsync("test", sourceContent).Wait();
destinationContent.Position = 0;
destination.UploadAsync("test", destinationContent).Wait();
var contentUpdate = new ContentUpdateWorkItem("test", "http://localhost:12345", sourceRfs.Storage,
sourceRfs.SigGenerator);
// force to upload entire file, we just want to check which pages will be reused
contentUpdate.UploadToAsync(destination.Synchronization).Wait();
destination.Synchronization.ResolveConflictAsync("test", ConflictResolutionStrategy.RemoteVersion).Wait();
contentUpdate.UploadToAsync(destination.Synchronization).Wait();
FileAndPages fileAndPages = null;
destinationRfs.Storage.Batch(accessor => fileAndPages = accessor.GetFile("test", 0, 2 * numberOfPages));
Assert.Equal(2 * numberOfPages, fileAndPages.Pages.Count);
for (var i = 0; i < numberOfPages; i++)
{
Assert.Equal(i + 1, fileAndPages.Pages[i].Id);
// if page ids are in the original order it means that they were used the existing pages
}
sourceContent.Position = 0;
Assert.Equal(sourceContent.GetMD5Hash(), destination.GetMetadataForAsync("test").Result["Content-MD5"]);
}
public async Task <SynchronizationReport> SynchronizeFileToAsync(string fileName, SynchronizationDestination destination)
{
ICredentials credentials = null;
if (string.IsNullOrEmpty(destination.Username) == false)
{
credentials = string.IsNullOrEmpty(destination.Domain)
? new NetworkCredential(destination.Username, destination.Password)
: new NetworkCredential(destination.Username, destination.Password, destination.Domain);
}
var destinationClient = new RavenFileSystemClient(destination.ServerUrl, destination.FileSystem, apiKey: destination.ApiKey, credentials: credentials).Synchronization;
RavenJObject destinationMetadata;
try
{
destinationMetadata = await destinationClient.GetMetadataForAsync(fileName);
}
catch (Exception ex)
{
var exceptionMessage = "Could not get metadata details for " + fileName + " from " + destination.FileSystemUrl;
Log.WarnException(exceptionMessage, ex);
return(new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown)
{
Exception = new SynchronizationException(exceptionMessage, ex)
});
}
RavenJObject localMetadata = GetLocalMetadata(fileName);
NoSyncReason reason;
SynchronizationWorkItem work = synchronizationStrategy.DetermineWork(fileName, localMetadata, destinationMetadata, FileSystemUrl, out reason);
if (work == null)
{
Log.Debug("File '{0}' was not synchronized to {1}. {2}", fileName, destination.FileSystemUrl, reason.GetDescription());
return(new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown)
{
Exception = new SynchronizationException(reason.GetDescription())
});
}
return(await PerformSynchronizationAsync(destinationClient, work));
}
public async Task Should_rename_file_on_all_destinations()
{
StartServerInstance(AddtitionalServerInstancePortNumber);
var sourceClient = NewClient(0);
var destination1Client = NewClient(1);
var destination2Client = new RavenFileSystemClient(ServerAddress(AddtitionalServerInstancePortNumber));
// upload file to all servers
await sourceClient.UploadAsync("test.bin", new RandomStream(10));
await sourceClient.Config.SetConfig(SynchronizationConstants.RavenSynchronizationDestinations, new NameValueCollection
{
{
"url", destination1Client.ServerUrl
},
{
"url", destination2Client.ServerUrl
}
});
await sourceClient.Synchronization.SynchronizeDestinationsAsync();
await sourceClient.Config.DeleteConfig(SynchronizationConstants.RavenSynchronizationDestinations);
// delete file on source
await sourceClient.RenameAsync("test.bin", "rename.bin");
// set up destinations
await sourceClient.Config.SetConfig(SynchronizationConstants.RavenSynchronizationDestinations, new NameValueCollection
{
{
"url", destination1Client.ServerUrl
},
{
"url", destination2Client.ServerUrl
}
});
var destinationSyncResults = sourceClient.Synchronization.SynchronizeDestinationsAsync().Result;
foreach (var destinationSyncResult in destinationSyncResults)
{
foreach (var report in destinationSyncResult.Reports)
{
Assert.Null(report.Exception);
Assert.Equal(SynchronizationType.Rename, report.Type);
}
}
Assert.Null(destination1Client.GetMetadataForAsync("test.bin").Result);
Assert.Null(destination2Client.GetMetadataForAsync("test.bin").Result);
Assert.NotNull(destination1Client.GetMetadataForAsync("rename.bin").Result);
Assert.NotNull(destination2Client.GetMetadataForAsync("rename.bin").Result);
}
public async Task Should_change_metadata_on_all_destinations()
{
StartServerInstance(AddtitionalServerInstancePortNumber);
var sourceClient = NewClient(0);
var destination1Client = NewClient(1);
var destination2Client = new RavenFileSystemClient(ServerAddress(AddtitionalServerInstancePortNumber));
var sourceContent = new MemoryStream();
var streamWriter = new StreamWriter(sourceContent);
var expected = new string('a', 1024*1024*10);
streamWriter.Write(expected);
streamWriter.Flush();
sourceContent.Position = 0;
await sourceClient.UploadAsync("test.txt", sourceContent);
await
sourceClient.Config.SetConfig(SynchronizationConstants.RavenSynchronizationDestinations, new NameValueCollection
{
{
"url", destination1Client.ServerUrl
},
{
"url", destination2Client.ServerUrl
}
});
// push file to all destinations
await sourceClient.Synchronization.SynchronizeDestinationsAsync();
// prevent pushing files after metadata update
await sourceClient.Config.DeleteConfig(SynchronizationConstants.RavenSynchronizationDestinations);
await sourceClient.UpdateMetadataAsync("test.txt", new NameValueCollection {{"value", "shouldBeSynchronized"}});
// add destinations again
await
sourceClient.Config.SetConfig(SynchronizationConstants.RavenSynchronizationDestinations, new NameValueCollection
{
{
"url", destination1Client.ServerUrl
},
{
"url", destination2Client.ServerUrl
}
});
// should synchronize metadata
var destinationSyncResults = sourceClient.Synchronization.SynchronizeDestinationsAsync().Result;
foreach (var destinationSyncResult in destinationSyncResults)
{
foreach (var report in destinationSyncResult.Reports)
{
Assert.Null(report.Exception);
Assert.Equal(SynchronizationType.MetadataUpdate, report.Type);
}
}
Assert.Equal("shouldBeSynchronized", destination1Client.GetMetadataForAsync("test.txt").Result["value"]);
Assert.Equal("shouldBeSynchronized", destination2Client.GetMetadataForAsync("test.txt").Result["value"]);
}
public async Task<SynchronizationReport> SynchronizeFileToAsync(string fileName, SynchronizationDestination destination)
{
ICredentials credentials = null;
if (string.IsNullOrEmpty(destination.Username) == false)
{
credentials = string.IsNullOrEmpty(destination.Domain)
? new NetworkCredential(destination.Username, destination.Password)
: new NetworkCredential(destination.Username, destination.Password, destination.Domain);
}
var destinationClient = new RavenFileSystemClient(destination.ServerUrl, destination.FileSystem, apiKey: destination.ApiKey, credentials: credentials).Synchronization;
RavenJObject destinationMetadata;
try
{
destinationMetadata = await destinationClient.GetMetadataForAsync(fileName);
}
catch (Exception ex)
{
var exceptionMessage = "Could not get metadata details for " + fileName + " from " + destination.FileSystemUrl;
Log.WarnException(exceptionMessage, ex);
return new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown)
{
Exception = new SynchronizationException(exceptionMessage, ex)
};
}
RavenJObject localMetadata = GetLocalMetadata(fileName);
NoSyncReason reason;
SynchronizationWorkItem work = synchronizationStrategy.DetermineWork(fileName, localMetadata, destinationMetadata, FileSystemUrl, out reason);
if (work == null)
{
Log.Debug("File '{0}' was not synchronized to {1}. {2}", fileName, destination.FileSystemUrl, reason.GetDescription());
return new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown)
{
Exception = new SynchronizationException(reason.GetDescription())
};
}
return await PerformSynchronizationAsync(destinationClient, work);
}
public async Task<SynchronizationReport> SynchronizeFileToAsync(string fileName, string destinationUrl)
{
var destinationClient = new RavenFileSystemClient(destinationUrl);
NameValueCollection destinationMetadata;
try
{
destinationMetadata = await destinationClient.GetMetadataForAsync(fileName);
}
catch (Exception ex)
{
var exceptionMessage = "Could not get metadata details for " + fileName + " from " + destinationUrl;
Log.WarnException(exceptionMessage, ex);
return new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown)
{
Exception = new SynchronizationException(exceptionMessage, ex)
};
}
NameValueCollection localMetadata = GetLocalMetadata(fileName);
NoSyncReason reason;
SynchronizationWorkItem work = synchronizationStrategy.DetermineWork(fileName, localMetadata, destinationMetadata,
ServerUrl, out reason);
if (work == null)
{
Log.Debug("File '{0}' was not synchronized to {1}. {2}", fileName, destinationUrl, reason.GetDescription());
return new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown)
{
Exception = new SynchronizationException(reason.GetDescription())
};
}
return await PerformSynchronizationAsync(destinationClient.ServerUrl, work);
}
private async Task EnqueueMissingUpdatesAsync(RavenFileSystemClient destinationClient,
SourceSynchronizationInformation lastEtag,
IList<FileHeader> needSyncingAgain)
{
LogFilesInfo("There were {0} file(s) that needed synchronization because the previous one went wrong: {1}",
needSyncingAgain);
var destinationUrl = destinationClient.ServerUrl;
var filesToSynchronization = new HashSet<FileHeader>(GetFilesToSynchronization(lastEtag, 100),
new FileHeaderNameEqualityComparer());
LogFilesInfo("There were {0} file(s) that needed synchronization because of greater ETag value: {1}",
filesToSynchronization);
foreach (FileHeader needSyncing in needSyncingAgain)
{
filesToSynchronization.Add(needSyncing);
}
var filteredFilesToSynchronization =
filesToSynchronization.Where(
x => synchronizationStrategy.Filter(x, lastEtag.DestinationServerId, filesToSynchronization)).ToList();
if (filesToSynchronization.Count > 0)
{
LogFilesInfo("There were {0} file(s) that needed synchronization after filtering: {1}",
filteredFilesToSynchronization);
}
if (filteredFilesToSynchronization.Count == 0)
return;
foreach (var fileHeader in filteredFilesToSynchronization)
{
var file = fileHeader.Name;
var localMetadata = GetLocalMetadata(file);
NameValueCollection destinationMetadata;
try
{
destinationMetadata = await destinationClient.GetMetadataForAsync(file);
}
catch (Exception ex)
{
Log.WarnException(
string.Format(
"Could not retrieve a metadata of a file '{0}' from {1} in order to determine needed synchronization type", file,
destinationUrl), ex);
continue;
}
NoSyncReason reason;
var work = synchronizationStrategy.DetermineWork(file, localMetadata, destinationMetadata, ServerUrl, out reason);
if (work == null)
{
Log.Debug("File '{0}' were not synchronized to {1}. {2}", file, destinationUrl, reason.GetDescription());
if (reason == NoSyncReason.ContainedInDestinationHistory)
{
var etag = localMetadata.Value<Guid>("ETag");
destinationClient.Synchronization.IncrementLastETagAsync(storage.Id, ServerUrl, etag);
RemoveSyncingConfiguration(file, destinationClient.ServerUrl);
}
continue;
}
synchronizationQueue.EnqueueSynchronization(destinationUrl, work);
}
}
public async Task Metadata_change_should_be_propagated()
{
StartServerInstance(AddtitionalServerInstancePortNumber);
var content = new MemoryStream(new byte[] {1, 2, 3});
var server1 = NewClient(0);
var server2 = NewClient(1);
var server3 = new RavenFileSystemClient(ServerAddress(AddtitionalServerInstancePortNumber));
content.Position = 0;
server1.UploadAsync("test.bin", new NameValueCollection {{"test", "value"}}, content).Wait();
SyncTestUtils.TurnOnSynchronization(server1, server2);
Assert.Null(server1.Synchronization.SynchronizeDestinationsAsync().Result[0].Exception);
SyncTestUtils.TurnOnSynchronization(server2, server3);
Assert.Null(server2.Synchronization.SynchronizeDestinationsAsync().Result[0].Exception);
SyncTestUtils.TurnOffSynchronization(server1);
server1.UpdateMetadataAsync("test.bin", new NameValueCollection {{"new_test", "new_value"}}).Wait();
SyncTestUtils.TurnOnSynchronization(server1, server2);
var secondServer1Synchronization = await server1.Synchronization.SynchronizeDestinationsAsync();
Assert.Null(secondServer1Synchronization[0].Exception);
Assert.Equal(SynchronizationType.MetadataUpdate, secondServer1Synchronization[0].Reports.ToArray()[0].Type);
var secondServer2Synchronization = await server2.Synchronization.SynchronizeDestinationsAsync();
Assert.Null(secondServer2Synchronization[0].Exception);
Assert.Equal(SynchronizationType.MetadataUpdate, secondServer2Synchronization[0].Reports.ToArray()[0].Type);
// On all servers should be file named "rename.bin"
var server1Metadata = server1.GetMetadataForAsync("test.bin").Result;
var server2Metadata = server2.GetMetadataForAsync("test.bin").Result;
var server3Metadata = server3.GetMetadataForAsync("test.bin").Result;
Assert.Equal("new_value", server1Metadata["new_test"]);
Assert.Equal("new_value", server2Metadata["new_test"]);
Assert.Equal("new_value", server3Metadata["new_test"]);
}
