public static SynchronizationDestination ToSynchronizationDestination(this RavenFileSystemClient self)
{
var result = new SynchronizationDestination()
{
FileSystem = self.FileSystemName,
ServerUrl = self.ServerUrl,
ApiKey = self.ApiKey
};
if (self.Credentials != null)
{
var networkCredential = self.Credentials as NetworkCredential;
if (networkCredential != null)
{
result.Username = networkCredential.UserName;
result.Password = networkCredential.Password;
result.Domain = networkCredential.Domain;
}
else
{
throw new InvalidOperationException("Expected NetworkCredential object while get: " + self.Credentials);
}
}
return result;
}
public static SynchronizationDestination ToSynchronizationDestination(this RavenFileSystemClient self)
{
var result = new SynchronizationDestination()
{
FileSystem = self.FileSystemName,
ServerUrl = self.ServerUrl,
ApiKey = self.ApiKey
};
if (self.Credentials != null)
{
var networkCredential = self.Credentials as NetworkCredential;
if (networkCredential != null)
{
result.Username = networkCredential.UserName;
result.Password = networkCredential.Password;
result.Domain = networkCredential.Domain;
}
else
{
throw new InvalidOperationException("Expected NetworkCredential object while get: " + self.Credentials);
}
}
return(result);
}
public async Task ShouldFailOver()
{
var sourceClient = NewClient(0);
var destinationClient = NewClient(1);
var source1Content = new RandomStream(10000);
await sourceClient.UploadAsync("test1.bin", source1Content);
var destination = new SynchronizationDestination()
{
FileSystem = destinationClient.FileSystemName,
ServerUrl = destinationClient.ServerUrl
};
await sourceClient.Config.SetDestinationsConfig(destination);
sourceClient.ReplicationInformer.RefreshReplicationInformation(sourceClient);
await sourceClient.Synchronization.SynchronizeDestinationsAsync();
var destinationFiles = await destinationClient.GetFilesAsync("/");
Assert.Equal(1, destinationFiles.FileCount);
Assert.Equal(1, destinationFiles.Files.Length);
var server = GetServer(0);
server.Dispose();
var fileFromSync = await sourceClient.GetFilesAsync("/");
Assert.Equal(1, fileFromSync.FileCount);
Assert.Equal(1, fileFromSync.Files.Length);
}
public async void MultipleConflictListeners_MultipleResolutionListeners()
{
var store = (FilesStore)filesStore;
var conflictsListener = new TakeLocalConflictListener();
var noOpListener = new NoOpConflictListener();
anotherStore.Listeners.RegisterListener(conflictsListener);
anotherStore.Listeners.RegisterListener(noOpListener);
using (var sessionDestination1 = filesStore.OpenAsyncSession())
using (var sessionDestination2 = anotherStore.OpenAsyncSession())
{
sessionDestination2.RegisterUpload("test1.file", CreateUniformFileStream(130));
await sessionDestination2.SaveChangesAsync();
sessionDestination1.RegisterUpload("test1.file", CreateUniformFileStream(128));
await sessionDestination1.SaveChangesAsync();
var syncDestinatios = new SynchronizationDestination[] { sessionDestination2.Commands.ToSynchronizationDestination() };
await sessionDestination1.Commands.Synchronization.SetDestinationsAsync(syncDestinatios);
await sessionDestination1.Commands.Synchronization.SynchronizeAsync();
Thread.Sleep(250);
Assert.Equal(1, conflictsListener.DetectedCount);
Assert.Equal(1, conflictsListener.ResolvedCount);
Assert.Equal(1, noOpListener.DetectedCount);
Assert.Equal(1, noOpListener.ResolvedCount);
}
}
public static SynchronizationDestination ToSynchronizationDestination(this IAsyncFilesCommands self)
{
var selfImpl = (IAsyncFilesCommandsImpl)self;
var result = new SynchronizationDestination
{
FileSystem = self.FileSystemName,
ServerUrl = selfImpl.ServerUrl,
};
if (self.PrimaryCredentials != null)
{
var networkCredential = self.PrimaryCredentials.Credentials as NetworkCredential;
if (networkCredential != null)
{
result.Username = networkCredential.UserName;
result.Password = networkCredential.Password;
result.Domain = networkCredential.Domain;
}
else
{
throw new InvalidOperationException("Expected NetworkCredential object while get: " + self.PrimaryCredentials.Credentials);
}
result.ApiKey = self.PrimaryCredentials.ApiKey;
}
return(result);
}
public Dictionary <SynchronizationDestination, Task <IEnumerable <Task <SynchronizationReport> > > > Execute(bool forceSyncingAll)
{
var destinationSyncs = new Dictionary <SynchronizationDestination, Task <IEnumerable <Task <SynchronizationReport> > > >();
foreach (var dst in GetSynchronizationDestinations())
{
SynchronizationDestination destination = dst;
// If the destination is disabled, we skip it.
if (destination.Enabled == false)
{
continue;
}
if (Log.IsDebugEnabled)
{
Log.Debug("Starting to synchronize a destination server {0}", dst.Url);
}
if (AvailableSynchronizationRequestsTo(destination.Url) <= 0)
{
if (Log.IsDebugEnabled)
{
Log.Debug("Could not synchronize to {0} because no synchronization request was available", dst.Url);
}
continue;
}
destinationSyncs.Add(destination, SynchronizeDestinationAsync(destination, forceSyncingAll));
}
return(destinationSyncs);
}
public async Task ConflictListeners_RemoteVersion()
{
var filename = FileHeader.Canonize("test1.file");
int firstStreamSize = 130;
int secondStreamSize = 128;
var store = this.NewStore(1);
var anotherStore = this.NewStore(2);
var conflictsListener = new TakeNewestConflictsListener();
anotherStore.Listeners.RegisterListener(conflictsListener);
using (var sessionDestination1 = store.OpenAsyncSession())
using (var sessionDestination2 = anotherStore.OpenAsyncSession())
{
sessionDestination2.RegisterUpload(filename, CreateUniformFileStream(firstStreamSize));
await sessionDestination2.SaveChangesAsync();
await sessionDestination1.Commands.Synchronization.SynchronizeAsync();
sessionDestination1.RegisterUpload(filename, CreateUniformFileStream(secondStreamSize));
await sessionDestination1.SaveChangesAsync();
var file = await sessionDestination1.LoadFileAsync(filename);
var file2 = await sessionDestination2.LoadFileAsync(filename);
Assert.Equal(secondStreamSize, file.TotalSize);
Assert.Equal(firstStreamSize, file2.TotalSize);
var notificationTask = await WaitForConflictResolved(anotherStore, 1, 10);
var syncDestinations = new SynchronizationDestination[] { sessionDestination2.Commands.ToSynchronizationDestination() };
await sessionDestination1.Commands.Synchronization.SetDestinationsAsync(syncDestinations);
await sessionDestination1.Commands.Synchronization.SynchronizeAsync();
//We need to sync again after conflict resolution because strategy was to resolve with remote
await sessionDestination1.Commands.Synchronization.SynchronizeAsync();
await notificationTask;
Assert.Equal(1, conflictsListener.DetectedCount);
Assert.Equal(1, conflictsListener.ResolvedCount);
file = await sessionDestination1.LoadFileAsync(filename);
file2 = await sessionDestination2.LoadFileAsync(filename);
Assert.Equal(secondStreamSize, file.TotalSize);
Assert.Equal(secondStreamSize, file2.TotalSize);
}
}
public async Task ConflictListeners_RemoteVersion()
{
var filename = FileHeader.Canonize("test1.file");
int firstStreamSize = 130;
int secondStreamSize = 128;
var store = this.NewStore(1);
var anotherStore = this.NewStore(2);
var conflictsListener = new TakeNewestConflictsListener();
anotherStore.Listeners.RegisterListener(conflictsListener);
using (var sessionDestination1 = store.OpenAsyncSession())
using (var sessionDestination2 = anotherStore.OpenAsyncSession())
{
sessionDestination2.RegisterUpload(filename, CreateUniformFileStream(firstStreamSize));
await sessionDestination2.SaveChangesAsync();
sessionDestination1.RegisterUpload(filename, CreateUniformFileStream(secondStreamSize));
await sessionDestination1.SaveChangesAsync();
var notificationTask = await WaitForConflictResolved(anotherStore, 1, 30);
var syncDestinations = new SynchronizationDestination[] { sessionDestination2.Commands.ToSynchronizationDestination() };
await sessionDestination1.Commands.Synchronization.SetDestinationsAsync(syncDestinations);
var syncResult = await sessionDestination1.Commands.Synchronization.StartAsync();
Assert.Equal(string.Format("File {0} is conflicted", filename), syncResult[0].Reports.ToList()[0].Exception.Message);
// conflict should be resolved by the registered listener
Assert.True(SpinWait.SpinUntil(() => conflictsListener.DetectedCount == 1 && conflictsListener.ResolvedCount == 1, TimeSpan.FromMinutes(1)),
string.Format("DetectedCount: {0}, ResolvedCount: {1}", conflictsListener.DetectedCount, conflictsListener.ResolvedCount));
// We need to sync again after conflict resolution because the strategy was to resolve with remote
await sessionDestination1.Commands.Synchronization.StartAsync();
await notificationTask;
Assert.Equal(1, conflictsListener.DetectedCount);
Assert.Equal(1, conflictsListener.ResolvedCount);
var file = await sessionDestination1.LoadFileAsync(filename);
var file2 = await sessionDestination2.LoadFileAsync(filename);
Assert.Equal(secondStreamSize, file.TotalSize);
Assert.Equal(secondStreamSize, file2.TotalSize);
}
}
public async Task MultipleConflictListeners_ConflictNotResolved()
{
var store = this.NewStore(1);
var anotherStore = this.NewStore(2);
var takeLocalConflictListener = new TakeLocalConflictListener();
var noOpListener = new NoOpConflictListener();
anotherStore.Listeners.RegisterListener(noOpListener);
anotherStore.Listeners.RegisterListener(takeLocalConflictListener);
using (var sessionDestination1 = store.OpenAsyncSession())
using (var sessionDestination2 = anotherStore.OpenAsyncSession())
{
sessionDestination2.RegisterUpload("test1.file", CreateUniformFileStream(130));
await sessionDestination2.SaveChangesAsync();
sessionDestination1.RegisterUpload("test1.file", CreateUniformFileStream(128));
await sessionDestination1.SaveChangesAsync();
var notificationTask = WaitForConflictDetected(anotherStore, 1, 10);
var syncDestinatios = new SynchronizationDestination[] { sessionDestination2.Commands.ToSynchronizationDestination() };
await sessionDestination1.Commands.Synchronization.SetDestinationsAsync(syncDestinatios);
await sessionDestination1.Commands.Synchronization.SynchronizeAsync();
await notificationTask;
Assert.Equal(1, noOpListener.DetectedCount);
Assert.Equal(1, takeLocalConflictListener.DetectedCount);
Assert.Equal(0, takeLocalConflictListener.ResolvedCount + noOpListener.ResolvedCount);
// try to change content of file in destination2
sessionDestination2.RegisterUpload("test1.file", CreateUniformFileStream(140));
// Assert an exception is thrown because the conflict is still there
var aggregateException = Assert.Throws <AggregateException>(() => sessionDestination2.SaveChangesAsync().Wait());
Assert.IsType <NotSupportedException>(aggregateException.InnerException);
// try to change content of file in destination2
sessionDestination2.RegisterRename("test1.file", "test2.file");
// Assert an exception is thrown because the conflict is still there
aggregateException = Assert.Throws <AggregateException>(() => sessionDestination2.SaveChangesAsync().Wait());
Assert.IsType <NotSupportedException>(aggregateException.InnerException);
}
}
public async Task <DestinationSyncResult> CreateDestinationResult(SynchronizationDestination destination, IEnumerable <Task <SynchronizationReport> > synchronizations)
{
try
{
var reports = await Task.WhenAll(synchronizations).ConfigureAwait(false);
var destinationSyncResult = new DestinationSyncResult
{
DestinationServer = destination.ServerUrl,
DestinationFileSystem = destination.FileSystem
};
if (reports.Length > 0)
{
var successfulSynchronizationsCount = reports.Count(x => x.Exception == null);
var failedSynchronizationsCount = reports.Count(x => x.Exception != null);
if (successfulSynchronizationsCount > 0 || failedSynchronizationsCount > 0)
{
if (Log.IsDebugEnabled)
{
Log.Debug(
"Synchronization to a destination {0} has completed. {1} file(s) were synchronized successfully, {2} synchronization(s) were failed",
destination.Url, successfulSynchronizationsCount, failedSynchronizationsCount);
}
}
destinationSyncResult.Reports = reports;
}
return(destinationSyncResult);
}
catch (Exception ex)
{
Log.WarnException(string.Format("Failed to perform a synchronization to a destination {0}", destination), ex);
return(new DestinationSyncResult
{
DestinationServer = destination.ServerUrl,
DestinationFileSystem = destination.FileSystem,
Exception = ex
});
}
}
public async void ConflictListeners_RemoteVersion()
{
var store = (FilesStore)filesStore;
var conflictsListener = new TakeNewestConflictsListener();
anotherStore.Listeners.RegisterListener(conflictsListener);
using (var sessionDestination1 = filesStore.OpenAsyncSession())
using (var sessionDestination2 = anotherStore.OpenAsyncSession())
{
sessionDestination2.RegisterUpload("test1.file", CreateUniformFileStream(130));
await sessionDestination2.SaveChangesAsync();
sessionDestination1.RegisterUpload("test1.file", CreateUniformFileStream(128));
await sessionDestination1.SaveChangesAsync();
var file = await sessionDestination1.LoadFileAsync("test1.file");
var file2 = await sessionDestination2.LoadFileAsync("test1.file");
Assert.Equal(128, file.TotalSize);
Assert.Equal(130, file2.TotalSize);
var syncDestinatios = new SynchronizationDestination[] { sessionDestination2.Commands.ToSynchronizationDestination() };
await sessionDestination1.Commands.Synchronization.SetDestinationsAsync(syncDestinatios);
await sessionDestination1.Commands.Synchronization.SynchronizeAsync();
Assert.Equal(1, conflictsListener.DetectedCount);
//We need to sync again after conflict resolution because strategy was to resolve with remote
await sessionDestination1.Commands.Synchronization.SynchronizeAsync();
Assert.Equal(1, conflictsListener.ResolvedCount);
file = await sessionDestination1.LoadFileAsync("test1.file");
file2 = await sessionDestination2.LoadFileAsync("test1.file");
Assert.Equal(128, file.TotalSize);
Assert.Equal(128, file2.TotalSize);
}
}
private IEnumerable <SynchronizationDetails> GetSyncingConfigurations(SynchronizationDestination destination)
{
var configObjects = new List <SynchronizationDetails>();
try
{
storage.Batch(
accessor =>
{
configObjects = accessor.GetConfigsStartWithPrefix(RavenFileNameHelper.SyncNameForFile(string.Empty, destination.Url), 0, 100)
.Select(config => config.JsonDeserialization <SynchronizationDetails>())
.ToList();
});
}
catch (Exception e)
{
Log.WarnException(string.Format("Could not get syncing configurations for a destination {0}", destination), e);
}
return(configObjects);
}
public async Task ConflictListeners_LocalVersion()
{
var store = this.NewStore(1);
var anotherStore = this.NewStore(2);
var conflictsListener = new TakeLocalConflictListener();
anotherStore.Listeners.RegisterListener(conflictsListener);
using (var sessionDestination1 = store.OpenAsyncSession())
using (var sessionDestination2 = anotherStore.OpenAsyncSession())
{
sessionDestination1.RegisterUpload("test1.file", CreateUniformFileStream(128));
await sessionDestination1.SaveChangesAsync();
sessionDestination2.RegisterUpload("test1.file", CreateUniformFileStream(130));
await sessionDestination2.SaveChangesAsync();
var notificationTask = await WaitForConflictResolved(anotherStore, 1, 10);
var syncDestinations = new SynchronizationDestination[] { sessionDestination2.Commands.ToSynchronizationDestination() };
await sessionDestination1.Commands.Synchronization.SetDestinationsAsync(syncDestinations);
await sessionDestination1.Commands.Synchronization.SynchronizeAsync();
await notificationTask;
Assert.Equal(1, conflictsListener.DetectedCount);
Assert.Equal(1, conflictsListener.ResolvedCount);
var file = await sessionDestination1.LoadFileAsync("test1.file");
var file2 = await sessionDestination2.LoadFileAsync("test1.file");
Assert.Equal(128, file.TotalSize);
Assert.Equal(130, file2.TotalSize);
}
}
public async Task MultipleConflictListeners_OnlyOneWithShortCircuitResolution()
{
var store = this.NewStore(1);
var anotherStore = this.NewStore(2);
var conflictsListener = new TakeLocalConflictListener();
var noOpListener = new NoOpConflictListener();
anotherStore.Listeners.RegisterListener(conflictsListener);
anotherStore.Listeners.RegisterListener(noOpListener);
using (var sessionDestination1 = store.OpenAsyncSession())
using (var sessionDestination2 = anotherStore.OpenAsyncSession())
{
sessionDestination2.RegisterUpload("test1.file", CreateUniformFileStream(130));
await sessionDestination2.SaveChangesAsync();
sessionDestination1.RegisterUpload("test1.file", CreateUniformFileStream(128));
await sessionDestination1.SaveChangesAsync();
var notificationTask = await WaitForConflictResolved(anotherStore, 1, 5);
var syncDestinatios = new SynchronizationDestination[] { sessionDestination2.Commands.ToSynchronizationDestination() };
await sessionDestination1.Commands.Synchronization.SetDestinationsAsync(syncDestinatios);
await sessionDestination1.Commands.Synchronization.SynchronizeAsync();
await notificationTask;
Assert.Equal(1, conflictsListener.DetectedCount);
Assert.Equal(1, conflictsListener.ResolvedCount);
Assert.Equal(0, noOpListener.DetectedCount);
Assert.Equal(1, noOpListener.ResolvedCount);
}
}
private IEnumerable <Task <SynchronizationReport> > SynchronizePendingFilesAsync(SynchronizationDestination destination, IAsyncFilesSynchronizationCommands destinationCommands, bool forceSyncingContinuation)
{
var commands = (IAsyncFilesCommandsImpl)destinationCommands.Commands;
var destinationUrl = commands.UrlFor();
for (var i = 0; i < AvailableSynchronizationRequestsTo(destinationUrl); i++)
{
SynchronizationWorkItem work;
if (!synchronizationQueue.TryDequePendingSynchronization(destinationUrl, out work))
{
break;
}
if (synchronizationQueue.IsDifferentWorkForTheSameFileBeingPerformed(work, destinationUrl))
{
Log.Debug("There was an already being performed synchronization of a file '{0}' to {1}", work.FileName,
destinationCommands);
if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work)) // add it again at the end of the queue
{
// add it again at the end of the queue
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Enqueue,
Direction = SynchronizationDirection.Outgoing
});
}
}
else
{
var workTask = PerformSynchronizationAsync(destinationCommands, work);
if (forceSyncingContinuation)
{
workTask.ContinueWith(async t =>
{
if (CanSynchronizeTo(destinationUrl))
{
await SynchronizeDestinationAsync(destination, true);
}
});
}
yield return(workTask);
}
}
}
private async Task <IEnumerable <Task <SynchronizationReport> > > SynchronizeDestinationAsync(SynchronizationDestination destination, bool forceSyncingAll)
{
ICredentials credentials = destination.Credentials;
var destinationSyncClient = new SynchronizationServerClient(destination.ServerUrl, destination.FileSystem, destination.ApiKey, credentials);
bool repeat;
do
{
var lastETag = await destinationSyncClient.GetLastSynchronizationFromAsync(storage.Id).ConfigureAwait(false);
var activeTasks = synchronizationQueue.Active;
var filesNeedConfirmation = GetSyncingConfigurations(destination).Where(sync => activeTasks.All(x => x.FileName != sync.FileName)).ToList();
var confirmations = await ConfirmPushedFiles(filesNeedConfirmation, destinationSyncClient).ConfigureAwait(false);
var needSyncingAgain = new List <FileHeader>();
Debug.Assert(filesNeedConfirmation.Count == confirmations.Length);
for (int i = 0; i < confirmations.Length; i++)
{
var confirmation = confirmations[i];
if (confirmation.Status == FileStatus.Safe)
{
if (Log.IsDebugEnabled)
{
Log.Debug("Destination server {0} said that file '{1}' is safe", destination, confirmation.FileName);
}
RemoveSyncingConfiguration(confirmation.FileName, destination.Url);
}
else
{
storage.Batch(accessor =>
{
var fileHeader = accessor.ReadFile(confirmation.FileName);
if (fileHeader == null)
{
if (Log.IsDebugEnabled)
{
Log.Debug("Destination server {0} said that file '{1}' is {2} but such file no longer exists. Removing related syncing configuration", destination, confirmation.FileName, confirmation.Status);
}
RemoveSyncingConfiguration(confirmation.FileName, destination.Url);
}
else if (EtagUtil.IsGreaterThan(fileHeader.Etag, filesNeedConfirmation[i].FileETag))
{
if (Log.IsDebugEnabled)
{
Log.Debug("Destination server {0} said that file '{1}' is {2} but such file has been changed since we stored the syncing configuration. " +
"Stored etag in configuration is {3} while current file etag is {4}. Removing related syncing configuration", destination, confirmation.FileName, confirmation.Status, fileHeader.Etag, filesNeedConfirmation[i].FileETag);
}
RemoveSyncingConfiguration(confirmation.FileName, destination.Url);
}
else
{
needSyncingAgain.Add(fileHeader);
if (Log.IsDebugEnabled)
{
Log.Debug("Destination server {0} said that file '{1}' is {2}.", destination, confirmation.FileName, confirmation.Status);
}
}
});
}
}
if (synchronizationQueue.NumberOfPendingSynchronizationsFor(destination.Url) < AvailableSynchronizationRequestsTo(destination.Url))
{
repeat = await EnqueueMissingUpdatesAsync(destinationSyncClient, lastETag, needSyncingAgain).ConfigureAwait(false) == false;
}
else
{
repeat = false;
}
}while (repeat);
return(SynchronizePendingFilesAsync(destinationSyncClient, forceSyncingAll));
}
protected bool Equals(SynchronizationDestination other)
{
return string.Equals(serverUrl, other.serverUrl) && string.Equals(ApiKey, other.ApiKey) && string.Equals(Domain, other.Domain) &&
string.Equals(Password, other.Password) && string.Equals(Username, other.Username) && string.Equals(FileSystem, other.FileSystem);
}
private SynchronizationTopologyDestinationNode HandleDestination(SynchronizationDestination synchronizationDestination)
{
var connectionStringOptions = new RavenConnectionStringOptions
{
Credentials = synchronizationDestination.Credentials,
ApiKey = synchronizationDestination.ApiKey,
Url = synchronizationDestination.ServerUrl,
DefaultDatabase = synchronizationDestination.FileSystem
};
string error;
string targetServerUrl;
var serverUrl = synchronizationDestination.Url;
// since each server can be addresses using both dns and ips we normalize connection string url by fetching target server url
// it should give us consistent urls
Guid?destinationFileSystemId;
if (FetchTargetServerUrl(serverUrl, connectionStringOptions,
out targetServerUrl, out destinationFileSystemId, out error) == false)
{
var offlineNode = SynchronizationTopologyDestinationNode.Offline(serverUrl, currentServerId, destinationFileSystemId);
if (string.IsNullOrEmpty(error) == false)
{
offlineNode.Errors.Add(error);
}
return(offlineNode);
}
if (synchronizationDestination.Enabled == false)
{
return(SynchronizationTopologyDestinationNode.Disabled(targetServerUrl, currentServerId, destinationFileSystemId));
}
if (from.Contains(targetServerUrl))
{
var state = CheckConnectionState(synchronizationDestination.Url, connectionStringOptions);
switch (state)
{
case ReplicatonNodeState.Online:
return(SynchronizationTopologyDestinationNode.Online(targetServerUrl, currentServerId, destinationFileSystemId));
case ReplicatonNodeState.Offline:
return(SynchronizationTopologyDestinationNode.Offline(targetServerUrl, currentServerId, destinationFileSystemId));
default:
throw new NotSupportedException(state.ToString());
}
}
SynchronizationTopologyRootNode rootNode;
if (TryGetSchema(targetServerUrl, connectionStringOptions, out rootNode, out error))
{
var node = SynchronizationTopologyDestinationNode.Online(targetServerUrl, currentServerId, destinationFileSystemId);
node.Destinations = rootNode.Destinations;
node.Sources = rootNode.Sources;
node.Errors = rootNode.Errors;
return(node);
}
var offline = SynchronizationTopologyDestinationNode.Offline(targetServerUrl, currentServerId, destinationFileSystemId);
if (string.IsNullOrEmpty(error) == false)
{
offline.Errors.Add(error);
}
return(offline);
}
public async Task <SynchronizationReport> SynchronizeFileToAsync(string fileName, SynchronizationDestination destination)
{
ICredentials credentials = null;
if (string.IsNullOrEmpty(destination.Username) == false)
{
credentials = string.IsNullOrEmpty(destination.Domain)
? new NetworkCredential(destination.Username, destination.Password)
: new NetworkCredential(destination.Username, destination.Password, destination.Domain);
}
var destinationClient = new AsyncFilesServerClient(destination.ServerUrl, destination.FileSystem, apiKey: destination.ApiKey, credentials: credentials).Synchronization;
RavenJObject destinationMetadata;
try
{
destinationMetadata = await destinationClient.Commands.GetMetadataForAsync(fileName);
}
catch (Exception ex)
{
var exceptionMessage = "Could not get metadata details for " + fileName + " from " + destination.Url;
Log.WarnException(exceptionMessage, ex);
return(new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown)
{
Exception = new SynchronizationException(exceptionMessage, ex)
});
}
RavenJObject localMetadata = GetLocalMetadata(fileName);
NoSyncReason reason;
SynchronizationWorkItem work = synchronizationStrategy.DetermineWork(fileName, localMetadata, destinationMetadata, FileSystemUrl, out reason);
if (work == null)
{
Log.Debug("File '{0}' was not synchronized to {1}. {2}", fileName, destination.Url, reason.GetDescription());
return(new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown)
{
Exception = new SynchronizationException(reason.GetDescription())
});
}
return(await PerformSynchronizationAsync(destinationClient, work));
}
protected bool Equals(SynchronizationDestination other)
{
return(string.Equals(serverUrl, other.serverUrl) && string.Equals(ApiKey, other.ApiKey) && string.Equals(Domain, other.Domain) &&
string.Equals(Password, other.Password) && string.Equals(Username, other.Username) && string.Equals(FileSystem, other.FileSystem));
}
private async Task <IEnumerable <Task <SynchronizationReport> > > SynchronizeDestinationAsync(SynchronizationDestination destination, bool forceSyncingAll)
{
ICredentials credentials = destination.Credentials;
var destinationSyncClient = new SynchronizationServerClient(destination.ServerUrl, destination.FileSystem, destination.ApiKey, credentials);
bool repeat;
do
{
var lastETag = await destinationSyncClient.GetLastSynchronizationFromAsync(storage.Id).ConfigureAwait(false);
var activeTasks = synchronizationQueue.Active;
var filesNeedConfirmation = GetSyncingConfigurations(destination).Where(sync => activeTasks.All(x => x.FileName != sync.FileName)).ToList();
var confirmations = await ConfirmPushedFiles(filesNeedConfirmation, destinationSyncClient).ConfigureAwait(false);
var needSyncingAgain = new List <FileHeader>();
foreach (var confirmation in confirmations)
{
if (confirmation.Status == FileStatus.Safe)
{
if (Log.IsDebugEnabled)
{
Log.Debug("Destination server {0} said that file '{1}' is safe", destination, confirmation.FileName);
}
RemoveSyncingConfiguration(confirmation.FileName, destination.Url);
}
else
{
storage.Batch(accessor =>
{
var fileHeader = accessor.ReadFile(confirmation.FileName);
if (fileHeader != null)
{
needSyncingAgain.Add(fileHeader);
if (Log.IsDebugEnabled)
{
Log.Debug("Destination server {0} said that file '{1}' is {2}.", destination, confirmation.FileName, confirmation.Status);
}
}
});
}
}
if (synchronizationQueue.NumberOfPendingSynchronizationsFor(destination.Url) < AvailableSynchronizationRequestsTo(destination.Url))
{
repeat = await EnqueueMissingUpdatesAsync(destinationSyncClient, lastETag, needSyncingAgain).ConfigureAwait(false) == false;
}
else
{
repeat = false;
}
}while (repeat);
return(SynchronizePendingFilesAsync(destinationSyncClient, forceSyncingAll));
}
public async Task <SynchronizationReport> SynchronizeFileToAsync(string fileName, SynchronizationDestination destination)
{
ICredentials credentials = destination.Credentials;
var conventions = new FilesConvention();
if (string.IsNullOrEmpty(destination.AuthenticationScheme) == false)
{
conventions.AuthenticationScheme = destination.AuthenticationScheme;
}
var destinationClient = new SynchronizationServerClient(destination.ServerUrl, destination.FileSystem, convention: conventions, apiKey: destination.ApiKey, credentials: credentials);
RavenJObject destinationMetadata;
try
{
destinationMetadata = await destinationClient.GetMetadataForAsync(fileName).ConfigureAwait(false);
}
catch (Exception ex)
{
var exceptionMessage = "Could not get metadata details for " + fileName + " from " + destination.Url;
Log.WarnException(exceptionMessage, ex);
return(new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown)
{
Exception = new SynchronizationException(exceptionMessage, ex)
});
}
RavenJObject localMetadata = GetLocalMetadata(fileName);
NoSyncReason reason;
SynchronizationWorkItem work = synchronizationStrategy.DetermineWork(fileName, localMetadata, destinationMetadata, FileSystemUrl, out reason);
if (work == null)
{
if (Log.IsDebugEnabled)
{
Log.Debug("File '{0}' was not synchronized to {1}. {2}", fileName, destination.Url, reason.GetDescription());
}
return(new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown)
{
Exception = new SynchronizationException(reason.GetDescription())
});
}
return(await PerformSynchronizationAsync(destinationClient, work).ConfigureAwait(false));
}
private async Task <DestinationSyncResult> SynchronizeDestinationAsync(SynchronizationDestination destination,
bool forceSyncingContinuation)
{
try
{
ICredentials credentials = null;
if (string.IsNullOrEmpty(destination.Username) == false)
{
credentials = string.IsNullOrEmpty(destination.Domain)
? new NetworkCredential(destination.Username, destination.Password)
: new NetworkCredential(destination.Username, destination.Password, destination.Domain);
}
var destinationClient = new AsyncFilesServerClient(destination.ServerUrl, destination.FileSystem,
apiKey: destination.ApiKey, credentials: credentials).Synchronization;
var lastETag = await destinationClient.GetLastSynchronizationFromAsync(storage.Id);
var activeTasks = synchronizationQueue.Active;
var filesNeedConfirmation = GetSyncingConfigurations(destination).Where(sync => activeTasks.All(x => x.FileName != sync.FileName)).ToList();
var confirmations = await ConfirmPushedFiles(filesNeedConfirmation, destinationClient);
var needSyncingAgain = new List <FileHeader>();
foreach (var confirmation in confirmations)
{
if (confirmation.Status == FileStatus.Safe)
{
Log.Debug("Destination server {0} said that file '{1}' is safe", destination, confirmation.FileName);
RemoveSyncingConfiguration(confirmation.FileName, destination.Url);
}
else
{
storage.Batch(accessor =>
{
var fileHeader = accessor.ReadFile(confirmation.FileName);
if (fileHeader != null)
{
needSyncingAgain.Add(fileHeader);
Log.Debug("Destination server {0} said that file '{1}' is {2}.", destination, confirmation.FileName, confirmation.Status);
}
});
}
}
await EnqueueMissingUpdatesAsync(destinationClient, lastETag, needSyncingAgain);
var reports = await Task.WhenAll(SynchronizePendingFilesAsync(destinationClient, forceSyncingContinuation));
var destinationSyncResult = new DestinationSyncResult
{
DestinationServer = destination.ServerUrl,
DestinationFileSystem = destination.FileSystem
};
if (reports.Length > 0)
{
var successfulSynchronizationsCount = reports.Count(x => x.Exception == null);
var failedSynchronizationsCount = reports.Count(x => x.Exception != null);
if (successfulSynchronizationsCount > 0 || failedSynchronizationsCount > 0)
{
Log.Debug(
"Synchronization to a destination {0} has completed. {1} file(s) were synchronized successfully, {2} synchronization(s) were failed",
destination.Url, successfulSynchronizationsCount, failedSynchronizationsCount);
}
destinationSyncResult.Reports = reports;
}
return(destinationSyncResult);
}
catch (Exception ex)
{
Log.WarnException(string.Format("Failed to perform a synchronization to a destination {0}", destination), ex);
return(new DestinationSyncResult
{
DestinationServer = destination.ServerUrl,
DestinationFileSystem = destination.FileSystem,
Exception = ex
});
}
}
