public static void TurnOnSynchronization(RavenFileSystemClient source, RavenFileSystemClient destination)
{
source.Config.SetConfig(SynchronizationConstants.RavenSynchronizationDestinations, new NameValueCollection
{
{"url", destination.ServerUrl},
}).Wait();
}
public RemoteRdcManager(RavenFileSystemClient ravenFileSystemClient, ISignatureRepository localSignatureRepository,
ISignatureRepository remoteCacheSignatureRepository)
{
this.localSignatureRepository = localSignatureRepository;
this.remoteCacheSignatureRepository = remoteCacheSignatureRepository;
this.ravenFileSystemClient = ravenFileSystemClient;
}
public static SynchronizationDestination ToSynchronizationDestination(this RavenFileSystemClient self)
{
var result = new SynchronizationDestination()
{
FileSystem = self.FileSystemName,
ServerUrl = self.ServerUrl,
ApiKey = self.ApiKey
};
if (self.Credentials != null)
{
var networkCredential = self.Credentials as NetworkCredential;
if (networkCredential != null)
{
result.Username = networkCredential.UserName;
result.Password = networkCredential.Password;
result.Domain = networkCredential.Domain;
}
else
{
throw new InvalidOperationException("Expected NetworkCredential object while get: " + self.Credentials);
}
}
return(result);
}
public override Task<SynchronizationReport> PerformAsync(RavenFileSystemClient.SynchronizationClient destination)
{
FileAndPages fileAndPages = null;
Storage.Batch(accessor => fileAndPages = accessor.GetFile(FileName, 0, 0));
return destination.DeleteAsync(FileName, fileAndPages.Metadata, ServerInfo);
}
public async Task CanCreateAndDeleteFileSystem()
{
var client = NewClient();
var adminClient = client.Admin;
const string newFileSystemName = "testName_CanDeleteFileSystem";
await adminClient.CreateOrUpdateFileSystemAsync(new DatabaseDocument
{
Id = "Raven/FileSystem/" + newFileSystemName,
Settings =
{
{ "Raven/FileSystem/DataDir", Path.Combine("~", Path.Combine("FileSystems", newFileSystemName)) }
}
}, newFileSystemName);
using (var createdFsClient = new RavenFileSystemClient(client.ServerUrl, newFileSystemName))
{
await createdFsClient.UploadAsync("foo", new MemoryStream(new byte[] { 1 }));
}
var names = await adminClient.GetFileSystemsNames();
Assert.Contains(newFileSystemName, names);
var stats = await adminClient.GetFileSystemsStats();
Assert.NotNull(stats.FirstOrDefault(x => x.Name == newFileSystemName));
await adminClient.DeleteFileSystemAsync(newFileSystemName);
names = await adminClient.GetFileSystemsNames();
Assert.DoesNotContain(newFileSystemName, names);
}
public async Task AdminClientWorkWithOAuthEnabled()
{
var client = NewClient(enableAuthentication: true, apiKey: apiKey);
var adminClient = client.Admin;
await adminClient.CreateFileSystemAsync(new DatabaseDocument
{
Id = "Raven/FileSystem/" + "testName",
Settings =
{
{ "Raven/FileSystem/DataDir", Path.Combine("~", Path.Combine("FileSystems", "testName")) }
}
}, "testName");
var names = await adminClient.GetFileSystemsNames();
Assert.Equal(1, names.Length); // will not return 'testName' file system name because used apiKey doesn't have access to a such file system
Assert.Equal("AdminClientWorkWithOAuthEnabled", names[0]);
var stats = await adminClient.GetFileSystemsStats();
Assert.Equal(0, stats.Count); // 0 because our fs aren't active
using (var createdFsClient = new RavenFileSystemClient(client.ServerUrl, "testName"))
{
await createdFsClient.UploadAsync("foo", new MemoryStream(new byte[] { 1 }));
}
await adminClient.DeleteFileSystemAsync("testName", true);
}
public async Task ShouldThrowWhenUsedApiKeyDefinitionDoesNotContainFileSystem()
{
var client = NewClient(enableAuthentication: true, apiKey: apiKey);
var server = GetServer();
await client.UploadAsync("abc.bin", new RandomStream(3));
using (var anotherClient = new RavenFileSystemClient(GetServerUrl(false, server.SystemDatabase.ServerUrl), "ShouldThrow_ApiKeyDoesnContainsThisFS", apiKey: apiKey))
{
await anotherClient.EnsureFileSystemExistsAsync(); // will pass because by using this api key we have access to <system> database
ErrorResponseException errorResponse = null;
try
{
await anotherClient.UploadAsync("def.bin", new RandomStream(1)); // should throw because a file system ShouldThrow_ApiKeyDoesnContainsThisFS isn't added to ApiKeyDefinition
}
catch (InvalidOperationException ex)
{
errorResponse = ex.InnerException as ErrorResponseException;
}
Assert.NotNull(errorResponse);
Assert.Equal(HttpStatusCode.Forbidden, errorResponse.StatusCode);
}
}
public async Task AdminClientWorkWithWinAuthEnabled()
{
var client = NewClient(enableAuthentication: true, credentials: new NetworkCredential(username, password, domain));
var adminClient = client.Admin;
await adminClient.CreateFileSystemAsync(new DatabaseDocument
{
Id = "Raven/FileSystem/" + "testName",
Settings =
{
{ "Raven/FileSystem/DataDir", Path.Combine("~", Path.Combine("FileSystems", "testName")) }
}
}, "testName");
using (var createdFsClient = new RavenFileSystemClient(client.ServerUrl, "testName", new NetworkCredential(username, password, domain)))
{
await createdFsClient.UploadAsync("foo", new MemoryStream(new byte[] { 1 }));
}
var names = await adminClient.GetFileSystemsNames();
Assert.Contains("testName", names);
var stats = await adminClient.GetFileSystemsStats();
Assert.NotNull(stats.FirstOrDefault(x => x.Name == "testName"));
await adminClient.DeleteFileSystemAsync("testName");
names = await adminClient.GetFileSystemsNames();
Assert.DoesNotContain("testName", names);
}
/// <summary>
/// Refreshes the replication information.
/// Expert use only.
/// </summary>
public async Task RefreshReplicationInformationAsync(RavenFileSystemClient serverClient)
{
var serverHash = ServerHash.GetServerHash(serverClient.ServerUrl);
try
{
var result = await serverClient.Config.GetConfig(SynchronizationConstants.RavenSynchronizationDestinations);
if (result == null)
{
LastReplicationUpdate = SystemTime.UtcNow; // checked and not found
}
else
{
var urls = result.GetValues("url");
replicationDestinations = urls == null ? new List <string>() : urls.ToList();
}
}
catch (Exception e)
{
log.ErrorException("Could not contact master for new replication information", e);
replicationDestinations = ReplicationInformerLocalCache.TryLoadReplicationInformationFromLocalCache(serverHash).ToList();
LastReplicationUpdate = SystemTime.UtcNow;
return;
}
failureCounts[serverClient.ServerUrl] = new FailureCounter(); // we just hit the master, so we can reset its failure count
ReplicationInformerLocalCache.TrySavingReplicationInformationToLocalCache(serverHash, replicationDestinations);
UpdateReplicationInformationFromDocument(replicationDestinations);
LastReplicationUpdate = SystemTime.UtcNow;
}
public async Task ShouldThrowWhenWindowsDocumentDoesNotContainFileSystem()
{
// in this test be careful if the specified credentials belong to admin user or not
var client = NewClient(enableAuthentication: true, credentials: new NetworkCredential(username, password, domain));
var server = GetServer();
await client.UploadAsync("abc.bin", new RandomStream(3));
using (var anotherClient = new RavenFileSystemClient(GetServerUrl(false, server.SystemDatabase.ServerUrl), "ShouldThrow_WindowsDocumentDoesnContainsThisFS",
credentials: new NetworkCredential(username, password, domain)))
{
await anotherClient.EnsureFileSystemExistsAsync(); // will pass because by using this api key we have access to <system> database
ErrorResponseException errorResponse = null;
try
{
await anotherClient.UploadAsync("def.bin", new RandomStream(1)); // should throw because a file system ShouldThrow_ApiKeyDoesnContainsThisFS isn't added to ApiKeyDefinition
}
catch (InvalidOperationException ex)
{
errorResponse = ex.InnerException as ErrorResponseException;
}
Assert.NotNull(errorResponse);
Assert.Equal(HttpStatusCode.Forbidden, errorResponse.StatusCode);
}
}
public async Task WillUseDefaultNetworkCredentialsWhenServerRequiresAuthentication()
{
var server = CreateRavenDbServer(Ports[0], fileSystemName: "WillUseDefaultCredentials", enableAuthentication: true); // enable authentication
using (var client = new RavenFileSystemClient(GetServerUrl(false, server.SystemDatabase.ServerUrl), "WillUseDefaultCredentials"))
{
await client.Admin.CreateFileSystemAsync(new DatabaseDocument()
{
Id = "Raven/FileSystem/" + client.FileSystemName,
Settings =
{
{ "Raven/FileSystem/DataDir", Path.Combine("FileSystems", client.FileSystemName) }
}
});
await client.UploadAsync("a", new MemoryStream(new byte[] { 1, 2 }));
var ms = new MemoryStream();
await client.DownloadAsync("a", ms);
var array = ms.ToArray();
Assert.Equal(1, array[0]);
Assert.Equal(2, array[1]);
}
}
public async Task Execute(string[] args)
{
var documentStore = new DocumentStore
{
Url = uri.ToString(),
}
.Initialize();
var client = new RavenFileSystemClient(uri.ToString(), filesystem);
Console.WriteLine("=== Available File Systems ===");
bool doesFileSystemExists = false;
foreach (string name in await client.Admin.GetFileSystemsNames())
{
Console.WriteLine(name);
doesFileSystemExists |= name.ToLower() == filesystem;
}
if (!doesFileSystemExists)
{
var filesystemDocument = new DatabaseDocument() { Id = "Raven/FileSystems/" + filesystem };
filesystemDocument.Settings["Raven/FileSystem/DataDir"] = Path.Combine(filesystemLocation, filesystem);
await client.Admin.CreateFileSystemAsync(filesystemDocument, filesystem);
}
Console.WriteLine();
foreach ( var file in directory.GetFiles())
{
await client.UploadAsync(file.Name, file.OpenRead());
}
}
static void DownloadFiles(RavenFileSystemClient client)
{
var files = Directory.GetFiles(@"C:\@DemoData");
Console.WriteLine("Downloading {0} files", files.Length);
foreach (var f in files)
{
Console.WriteLine();
var filename = Path.GetFileName(f);
try
{
var sw = new Stopwatch();
sw.Start();
var stream = File.Create(Path.Combine(@"C:\@DemoFiles", filename));
Console.WriteLine("** Downloading: {0}", filename);
client.DownloadAsync(filename, stream).Wait();
stream.Close();
stream.Dispose();
sw.Stop();
Console.WriteLine("** Done: {0}: Seconds {1}", filename,
(sw.ElapsedMilliseconds / 1000));
}
catch (Exception ex)
{
Console.WriteLine(ex.InnerException);
}
}
}
static void UploadFiles(RavenFileSystemClient client)
{
var files = Directory.GetFiles(@"C:\@DemoData");
Console.WriteLine("Uploading {0} files", files.Length);
foreach (var f in files)
{
Console.WriteLine();
var filename = Path.GetFileName(f);
var fileInfo = new System.IO.FileInfo(f);
try
{
var sw = new Stopwatch();
sw.Start();
Console.WriteLine("** Uploading: {0}, Size: {1} KB", filename, (fileInfo.Length / 1024));
client.UploadAsync(filename, new NameValueCollection(), File.OpenRead(f)).Wait();
sw.Stop();
Console.WriteLine("** Done: {0}: Seconds {1}", filename,
(sw.ElapsedMilliseconds / 1000));
}
catch (Exception ex)
{
Console.WriteLine(ex.InnerException);
}
}
}
public async Task WillUseDefaultNetworkCredentialsWhenServerRequiresAuthentication()
{
var server = CreateRavenDbServer(Ports[0], fileSystemName: "WillUseDefaultCredentials", enableAuthentication: true); // enable authentication
using (var client = new RavenFileSystemClient(GetServerUrl(false, server.SystemDatabase.ServerUrl), "WillUseDefaultCredentials"))
{
await client.Admin.CreateFileSystemAsync(new DatabaseDocument()
{
Id = "Raven/FileSystem/" + client.FileSystemName,
Settings =
{
{"Raven/FileSystem/DataDir", Path.Combine("~", Path.Combine("FileSystems", client.FileSystemName))}
}
});
await client.UploadAsync("a", new NameValueCollection(), new MemoryStream(new byte[] {1, 2}));
var ms = new MemoryStream();
await client.DownloadAsync("a", ms);
var array = ms.ToArray();
Assert.Equal(1, array[0]);
Assert.Equal(2, array[1]);
}
}
public async Task Can_get_stats_for_all_active_file_systems()
{
var client = NewClient();
var server = GetServer();
using (var anotherClient = new RavenFileSystemClient(GetServerUrl(false, server.SystemDatabase.ServerUrl), "test"))
{
await anotherClient.EnsureFileSystemExistsAsync();
await client.UploadAsync("test1", new RandomStream(10)); // will make it active
await anotherClient.UploadAsync("test1", new RandomStream(10)); // will make it active
await client.UploadAsync("test2", new RandomStream(10));
var stats = await anotherClient.Admin.GetFileSystemsStats();
var stats1 = stats.FirstOrDefault(x => x.Name == client.FileSystemName);
Assert.NotNull(stats1);
var stats2 = stats.FirstOrDefault(x => x.Name == anotherClient.FileSystemName);
Assert.NotNull(stats2);
Assert.Equal(2, stats1.Metrics.Requests.Count);
Assert.Equal(1, stats2.Metrics.Requests.Count);
Assert.Equal(0, stats1.ActiveSyncs.Count);
Assert.Equal(0, stats1.PendingSyncs.Count);
Assert.Equal(0, stats2.ActiveSyncs.Count);
Assert.Equal(0, stats2.PendingSyncs.Count);
}
}
protected RavenFileSystemClient CreateFileSystemClient(RavenConnectionStringOptions options, string fileSystemName)
{
var fsClient = new RavenFileSystemClient(options.Url, fileSystemName, apiKey: options.ApiKey, credentials: options.Credentials);
ValidateThatServerIsUpAndFileSystemExists(fsClient);
return(fsClient);
}
public ApplicationModel()
{
Notifications = new ObservableCollection<Notification>();
Notifications.CollectionChanged += delegate { OnPropertyChanged(() => ErrorCount); };
Client = new RavenFileSystemClient(DetermineUri()) {IsObservingFailedUploads = true};
AsyncOperations = new AsyncOperationsModel();
State = new ApplicationState();
}
public static void TurnOnSynchronization(RavenFileSystemClient source, params RavenFileSystemClient[] destinations)
{
source.Config.SetDestinationsConfig(destinations.Select(x => new SynchronizationDestination()
{
FileSystem = x.FileSystemName,
ServerUrl = x.ServerUrl
}).ToArray()).Wait();
}
public SynchronizationStorageTests()
{
source = NewClient(0);
destination = NewClient(1);
sourceRfs = GetRavenFileSystem(0);
destinationRfs = GetRavenFileSystem(1);
}
protected RavenFileSystemClient CreateFileSystemClient(RavenConnectionStringOptions options, string fileSystemName)
{
var fsClient = new RavenFileSystemClient(options.Url, fileSystemName, apiKey: options.ApiKey, credentials: options.Credentials);
ValidateThatServerIsUpAndFileSystemExists(fsClient);
return fsClient;
}
private static void ZeroTimeoutTest(RavenFileSystemClient destinationClient, Action action)
{
destinationClient.Config.SetConfig(RavenFileNameHelper.SyncLockNameForFile("test.bin"), SynchronizationConfig(DateTime.MinValue)).Wait();
destinationClient.Config.SetConfig(SynchronizationConstants.RavenSynchronizationLockTimeout, TimeSpan.FromSeconds(0)).Wait();
Assert.DoesNotThrow(() => action());
}
public static void TurnOnSynchronization(RavenFileSystemClient source, params RavenFileSystemClient[] destinations)
{
source.Config.SetDestinationsConfig(destinations.Select(x => new SynchronizationDestination()
{
FileSystem = x.FileSystemName,
ServerUrl = x.ServerUrl
}).ToArray()).Wait();
}
public SynchronizationStorageTests()
{
source = NewClient(0);
destination = NewClient(1);
sourceRfs = GetRavenFileSystem(0);
destinationRfs = GetRavenFileSystem(1);
}
public override async Task<SynchronizationReport> PerformAsync(string destination)
{
AssertLocalFileExistsAndIsNotConflicted(FileMetadata);
var destinationRavenFileSystemClient = new RavenFileSystemClient(destination);
var destinationMetadata = await destinationRavenFileSystemClient.GetMetadataForAsync(FileName);
if (destinationMetadata == null)
{
// if file doesn't exist on destination server - upload it there
return await UploadToAsync(destination);
}
var destinationServerRdcStats = await destinationRavenFileSystemClient.Synchronization.GetRdcStatsAsync();
if (!IsRemoteRdcCompatible(destinationServerRdcStats))
throw new SynchronizationException("Incompatible RDC version detected on destination server");
var conflict = CheckConflictWithDestination(FileMetadata, destinationMetadata, ServerInfo.Url);
if (conflict != null)
return await ApplyConflictOnDestinationAsync(conflict, destination, ServerInfo.Url, log);
using (var localSignatureRepository = new StorageSignatureRepository(Storage, FileName))
using (var remoteSignatureCache = new VolatileSignatureRepository(FileName))
{
var localRdcManager = new LocalRdcManager(localSignatureRepository, Storage, sigGenerator);
var destinationRdcManager = new RemoteRdcManager(destinationRavenFileSystemClient, localSignatureRepository,
remoteSignatureCache);
log.Debug("Starting to retrieve signatures of a local file '{0}'.", FileName);
Cts.Token.ThrowIfCancellationRequested();
// first we need to create a local file signatures before we synchronize with remote ones
var localSignatureManifest = await localRdcManager.GetSignatureManifestAsync(FileDataInfo);
log.Debug("Number of a local file '{0}' signatures was {1}.", FileName, localSignatureManifest.Signatures.Count);
if (localSignatureManifest.Signatures.Count > 0)
{
var destinationSignatureManifest = await destinationRdcManager.SynchronizeSignaturesAsync(FileDataInfo, Cts.Token);
if (destinationSignatureManifest.Signatures.Count > 0)
{
return
await
SynchronizeTo(destination, localSignatureRepository, remoteSignatureCache, localSignatureManifest,
destinationSignatureManifest);
}
}
return await UploadToAsync(destination);
}
}
public static SynchronizationReport ResolveConflictAndSynchronize(RavenFileSystemClient sourceClient,
RavenFileSystemClient destinationClient,
string fileName)
{
var shouldBeConflict = sourceClient.Synchronization.StartAsync(fileName, destinationClient).Result;
Assert.NotNull(shouldBeConflict.Exception);
destinationClient.Synchronization.ResolveConflictAsync(fileName, ConflictResolutionStrategy.RemoteVersion).Wait();
return(sourceClient.Synchronization.StartAsync(fileName, destinationClient).Result);
}
public static SynchronizationReport ResolveConflictAndSynchronize(RavenFileSystemClient sourceClient,
RavenFileSystemClient destinationClient,
string fileName)
{
var shouldBeConflict = sourceClient.Synchronization.StartAsync(fileName, destinationClient).Result;
Assert.NotNull(shouldBeConflict.Exception);
destinationClient.Synchronization.ResolveConflictAsync(fileName, ConflictResolutionStrategy.RemoteVersion).Wait();
return sourceClient.Synchronization.StartAsync(fileName, destinationClient).Result;
}
static void Main(string[] args)
{
//var fs = new RavenFileSystemClient("http://storage.wolfeautomation.com/");
// var fs = new RavenFileSystemClient("http://localfs/");
var fs = new RavenFileSystemClient("http://reduction:9090/");
SimpleTest(fs);
// UploadFiles(fs);
//DownloadFiles(fs);
}
public override Task<SynchronizationReport> PerformAsync(RavenFileSystemClient.SynchronizationClient destination)
{
AssertLocalFileExistsAndIsNotConflicted(FileMetadata);
var conflict = CheckConflictWithDestination(FileMetadata, destinationMetadata, ServerInfo.FileSystemUrl);
if (conflict != null)
return ApplyConflictOnDestinationAsync(conflict, destination, ServerInfo.FileSystemUrl, log);
return destination.UpdateMetadataAsync(FileName, FileMetadata, ServerInfo);
}
public SynchronizationMultipartRequest(RavenFileSystemClient.SynchronizationClient destination, ServerInfo serverInfo, string fileName,
RavenJObject sourceMetadata, Stream sourceStream, IList<RdcNeed> needList)
{
this.destination = destination;
this.serverInfo = serverInfo;
this.fileName = fileName;
this.sourceMetadata = sourceMetadata;
this.sourceStream = sourceStream;
this.needList = needList;
syncingBoundary = "syncing";
}
private void UploadFilesSynchronously(out RavenFileSystemClient sourceClient,
out RavenFileSystemClient destinationClient, string fileName = "test.bin")
{
sourceClient = NewClient(1);
destinationClient = NewClient(0);
var sourceContent = new RandomlyModifiedStream(new RandomStream(10, 1), 0.01);
var destinationContent = new RandomlyModifiedStream(new RandomStream(10, 1), 0.01);
destinationClient.UploadAsync(fileName, destinationContent).Wait();
sourceClient.UploadAsync(fileName, sourceContent).Wait();
}
public Task UpdateReplicationInformationIfNeeded(RavenFileSystemClient serverClient)
#endif
{
if (Conventions.FailoverBehavior == FailoverBehavior.FailImmediately)
{
return(new CompletedTask());
}
if (LastReplicationUpdate.AddMinutes(5) > SystemTime.UtcNow)
{
return(new CompletedTask());
}
lock (replicationLock)
{
if (firstTime)
{
var serverHash = ServerHash.GetServerHash(serverClient.ServerUrl);
var destinations = ReplicationInformerLocalCache.TryLoadReplicationInformationFromLocalCache(serverHash);
if (destinations != null)
{
UpdateReplicationInformationFromDocument(destinations);
}
}
firstTime = false;
if (LastReplicationUpdate.AddMinutes(5) > SystemTime.UtcNow)
{
return(new CompletedTask());
}
var taskCopy = refreshReplicationInformationTask;
if (taskCopy != null)
{
return(taskCopy);
}
return(refreshReplicationInformationTask = RefreshReplicationInformationAsync(serverClient)
.ContinueWith(task =>
{
if (task.Exception != null)
{
log.ErrorException("Failed to refresh replication information", task.Exception);
}
refreshReplicationInformationTask = null;
}));
}
}
protected virtual RavenFileSystemClient NewClient(int index = 0, bool fiddler = false, bool enableAuthentication = false, string apiKey = null,
ICredentials credentials = null, string requestedStorage = null, [CallerMemberName] string fileSystemName = null)
{
fileSystemName = NormalizeFileSystemName(fileSystemName);
var server = CreateRavenDbServer(Ports[index], fileSystemName: fileSystemName, enableAuthentication: enableAuthentication, requestedStorage: requestedStorage);
var client = new RavenFileSystemClient(GetServerUrl(fiddler, server.SystemDatabase.ServerUrl), fileSystemName, apiKey: apiKey, credentials: credentials);
client.EnsureFileSystemExistsAsync().Wait();
ravenFileSystemClients.Add(client);
return(client);
}
public async Task <SynchronizationReport> SynchronizeFileToAsync(string fileName, SynchronizationDestination destination)
{
ICredentials credentials = null;
if (string.IsNullOrEmpty(destination.Username) == false)
{
credentials = string.IsNullOrEmpty(destination.Domain)
? new NetworkCredential(destination.Username, destination.Password)
: new NetworkCredential(destination.Username, destination.Password, destination.Domain);
}
var destinationClient = new RavenFileSystemClient(destination.ServerUrl, destination.FileSystem, apiKey: destination.ApiKey, credentials: credentials).Synchronization;
RavenJObject destinationMetadata;
try
{
destinationMetadata = await destinationClient.GetMetadataForAsync(fileName);
}
catch (Exception ex)
{
var exceptionMessage = "Could not get metadata details for " + fileName + " from " + destination.FileSystemUrl;
Log.WarnException(exceptionMessage, ex);
return(new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown)
{
Exception = new SynchronizationException(exceptionMessage, ex)
});
}
RavenJObject localMetadata = GetLocalMetadata(fileName);
NoSyncReason reason;
SynchronizationWorkItem work = synchronizationStrategy.DetermineWork(fileName, localMetadata, destinationMetadata, FileSystemUrl, out reason);
if (work == null)
{
Log.Debug("File '{0}' was not synchronized to {1}. {2}", fileName, destination.FileSystemUrl, reason.GetDescription());
return(new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown)
{
Exception = new SynchronizationException(reason.GetDescription())
});
}
return(await PerformSynchronizationAsync(destinationClient, work));
}
public async Task File_rename_should_be_propagated()
{
StartServerInstance(AddtitionalServerInstancePortNumber);
var content = new MemoryStream(new byte[] {1, 2, 3});
var server1 = NewClient(0);
var server2 = NewClient(1);
var server3 = new RavenFileSystemClient(ServerAddress(AddtitionalServerInstancePortNumber));
content.Position = 0;
server1.UploadAsync("test.bin", new NameValueCollection {{"test", "value"}}, content).Wait();
SyncTestUtils.TurnOnSynchronization(server1, server2);
Assert.Null(server1.Synchronization.SynchronizeDestinationsAsync().Result[0].Exception);
SyncTestUtils.TurnOnSynchronization(server2, server3);
Assert.Null(server2.Synchronization.SynchronizeDestinationsAsync().Result[0].Exception);
SyncTestUtils.TurnOffSynchronization(server1);
server1.RenameAsync("test.bin", "rename.bin").Wait();
SyncTestUtils.TurnOnSynchronization(server1, server2);
var secondServer1Synchronization = await server1.Synchronization.SynchronizeDestinationsAsync();
Assert.Null(secondServer1Synchronization[0].Exception);
Assert.Equal(SynchronizationType.Rename, secondServer1Synchronization[0].Reports.ToArray()[0].Type);
var secondServer2Synchronization = await server2.Synchronization.SynchronizeDestinationsAsync();
Assert.Null(secondServer2Synchronization[0].Exception);
Assert.Equal(SynchronizationType.Rename, secondServer2Synchronization[0].Reports.ToArray()[0].Type);
// On all servers should be file named "rename.bin"
Assert.Equal(1, server1.BrowseAsync().Result.Count());
Assert.Equal("rename.bin", server1.BrowseAsync().Result[0].Name);
Assert.Equal(1, server2.BrowseAsync().Result.Count());
Assert.Equal("rename.bin", server2.BrowseAsync().Result[0].Name);
Assert.Equal(1, server3.BrowseAsync().Result.Count());
Assert.Equal("rename.bin", server3.BrowseAsync().Result[0].Name);
}
static void SimpleTest(RavenFileSystemClient fs)
{
var ms = new MemoryStream();
var streamWriter = new StreamWriter(ms);
var expected = new string('a', 1024);
streamWriter.Write(expected);
streamWriter.Flush();
ms.Position = 0;
fs.UploadAsync("abc.txt", ms).Wait();
var ms2 = new MemoryStream();
fs.DownloadAsync("abc.txt", ms2).Wait();
ms2.Position = 0;
var actual = new StreamReader(ms2).ReadToEnd();
Console.WriteLine(actual.Length);
}
public static async Task EnsureFileSystemExistsAsync(this RavenFileSystemClient client)
{
var existingSystems = await client.Admin.GetFileSystemsNames();
if (existingSystems.Any(x => x.Equals(client.FileSystemName, StringComparison.InvariantCultureIgnoreCase)))
{
return;
}
await client.Admin.CreateFileSystemAsync(new DatabaseDocument
{
Id = "Raven/FileSystem/" + client.FileSystemName,
Settings =
{
{ "Raven/FileSystem/DataDir", Path.Combine("FileSystems", client.FileSystemName) }
}
});
}
protected void ValidateThatServerIsUpAndFileSystemExists(RavenFileSystemClient fsClient)
{
var shouldDispose = false;
try
{
var fileSystemStats = fsClient.StatsAsync().Result;
}
catch (Exception e)
{
shouldDispose = true;
var responseException = e as ErrorResponseException;
if (responseException != null && responseException.StatusCode == HttpStatusCode.ServiceUnavailable && responseException.Message.StartsWith("Could not find a file system named:"))
{
throw new InvalidOperationException(
string.Format(
"Migration tool does not support file system creation (file system '{0}' on server '{1}' must exist before running this tool).",
fsClient.FileSystemName,
fsClient.ServerUrl), e);
}
if (e.InnerException != null)
{
var webException = e.InnerException as WebException;
if (webException != null)
{
throw new InvalidOperationException(string.Format("Migration tool encountered a connection problem: '{0}'.", webException.Message), webException);
}
}
throw new InvalidOperationException(string.Format("Migration tool encountered a connection problem: '{0}'.", e.Message), e);
}
finally
{
if (shouldDispose)
{
fsClient.Dispose();
}
}
}
public async Task<SynchronizationReport> SynchronizeFileToAsync(string fileName, string destinationUrl)
{
var destinationClient = new RavenFileSystemClient(destinationUrl);
NameValueCollection destinationMetadata;
try
{
destinationMetadata = await destinationClient.GetMetadataForAsync(fileName);
}
catch (Exception ex)
{
var exceptionMessage = "Could not get metadata details for " + fileName + " from " + destinationUrl;
Log.WarnException(exceptionMessage, ex);
return new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown)
{
Exception = new SynchronizationException(exceptionMessage, ex)
};
}
NameValueCollection localMetadata = GetLocalMetadata(fileName);
NoSyncReason reason;
SynchronizationWorkItem work = synchronizationStrategy.DetermineWork(fileName, localMetadata, destinationMetadata,
ServerUrl, out reason);
if (work == null)
{
Log.Debug("File '{0}' was not synchronized to {1}. {2}", fileName, destinationUrl, reason.GetDescription());
return new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown)
{
Exception = new SynchronizationException(reason.GetDescription())
};
}
return await PerformSynchronizationAsync(destinationClient.ServerUrl, work);
}
public async Task Execute(string[] args)
{
var documentStore = new DocumentStore
{
Url = uri.ToString(),
}
.Initialize();
var client = new RavenFileSystemClient(uri.ToString(), filesystem);
Console.WriteLine("=== Available File Systems ===");
bool doesFileSystemExists = false;
foreach (string name in await client.Admin.GetFileSystemsNames())
{
Console.WriteLine(name);
doesFileSystemExists |= name.ToLower() == filesystem;
}
if (!doesFileSystemExists)
{
var filesystemDocument = new DatabaseDocument()
{
Id = "Raven/FileSystems/" + filesystem
};
filesystemDocument.Settings["Raven/FileSystem/DataDir"] = Path.Combine(filesystemLocation, filesystem);
await client.Admin.CreateFileSystemAsync(filesystemDocument, filesystem);
}
Console.WriteLine();
foreach (var file in directory.GetFiles())
{
await client.UploadAsync(file.Name, file.OpenRead());
}
}
public async Task<SynchronizationReport> SynchronizeFileToAsync(string fileName, SynchronizationDestination destination)
{
ICredentials credentials = null;
if (string.IsNullOrEmpty(destination.Username) == false)
{
credentials = string.IsNullOrEmpty(destination.Domain)
? new NetworkCredential(destination.Username, destination.Password)
: new NetworkCredential(destination.Username, destination.Password, destination.Domain);
}
var destinationClient = new RavenFileSystemClient(destination.ServerUrl, destination.FileSystem, apiKey: destination.ApiKey, credentials: credentials).Synchronization;
RavenJObject destinationMetadata;
try
{
destinationMetadata = await destinationClient.GetMetadataForAsync(fileName);
}
catch (Exception ex)
{
var exceptionMessage = "Could not get metadata details for " + fileName + " from " + destination.FileSystemUrl;
Log.WarnException(exceptionMessage, ex);
return new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown)
{
Exception = new SynchronizationException(exceptionMessage, ex)
};
}
RavenJObject localMetadata = GetLocalMetadata(fileName);
NoSyncReason reason;
SynchronizationWorkItem work = synchronizationStrategy.DetermineWork(fileName, localMetadata, destinationMetadata, FileSystemUrl, out reason);
if (work == null)
{
Log.Debug("File '{0}' was not synchronized to {1}. {2}", fileName, destination.FileSystemUrl, reason.GetDescription());
return new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown)
{
Exception = new SynchronizationException(reason.GetDescription())
};
}
return await PerformSynchronizationAsync(destinationClient, work);
}
private async Task<DestinationSyncResult> SynchronizeDestinationAsync(SynchronizationDestination destination,
bool forceSyncingContinuation)
{
try
{
ICredentials credentials = null;
if (string.IsNullOrEmpty(destination.Username) == false)
{
credentials = string.IsNullOrEmpty(destination.Domain)
? new NetworkCredential(destination.Username, destination.Password)
: new NetworkCredential(destination.Username, destination.Password, destination.Domain);
}
var destinationClient = new RavenFileSystemClient(destination.ServerUrl, destination.FileSystem,
apiKey: destination.ApiKey, credentials: credentials).Synchronization;
var lastETag = await destinationClient.GetLastSynchronizationFromAsync(storage.Id);
var activeTasks = synchronizationQueue.Active.ToList();
var filesNeedConfirmation = GetSyncingConfigurations(destination).Where(sync => activeTasks.All(x => x.FileName != sync.FileName)).ToList();
var confirmations = await ConfirmPushedFiles(filesNeedConfirmation, destinationClient);
var needSyncingAgain = new List<FileHeader>();
foreach (var confirmation in confirmations)
{
if (confirmation.Status == FileStatus.Safe)
{
Log.Debug("Destination server {0} said that file '{1}' is safe", destination, confirmation.FileName);
RemoveSyncingConfiguration(confirmation.FileName, destination.FileSystemUrl);
}
else
{
storage.Batch(accessor =>
{
var fileHeader = accessor.ReadFile(confirmation.FileName);
if (fileHeader != null)
{
needSyncingAgain.Add(fileHeader);
Log.Debug("Destination server {0} said that file '{1}' is {2}.",
destination, confirmation.FileName, confirmation.Status);
}
});
}
}
await EnqueueMissingUpdatesAsync(destinationClient, lastETag, needSyncingAgain);
var reports = await Task.WhenAll(SynchronizePendingFilesAsync(destinationClient, forceSyncingContinuation));
var destinationSyncResult = new DestinationSyncResult
{
DestinationServer = destination.ServerUrl,
DestinationFileSystem = destination.FileSystem
};
if (reports.Length > 0)
{
var successfulSynchronizationsCount = reports.Count(x => x.Exception == null);
var failedSynchronizationsCount = reports.Count(x => x.Exception != null);
if (successfulSynchronizationsCount > 0 || failedSynchronizationsCount > 0)
{
Log.Debug(
"Synchronization to a destination {0} has completed. {1} file(s) were synchronized successfully, {2} synchronization(s) were failed",
destination.FileSystemUrl, successfulSynchronizationsCount, failedSynchronizationsCount);
}
destinationSyncResult.Reports = reports;
}
return destinationSyncResult;
}
catch (Exception ex)
{
Log.WarnException(string.Format("Failed to perform a synchronization to a destination {0}", destination), ex);
return new DestinationSyncResult
{
DestinationServer = destination.ServerUrl,
DestinationFileSystem = destination.FileSystem,
Exception = ex
};
}
}
public Notifications()
{
client = NewClient();
}
public async Task ShouldThrowWhenUsedApiKeyDefinitionDoesNotContainFileSystem()
{
var client = NewClient(enableAuthentication: true, apiKey: apiKey);
var server = GetServer();
await client.UploadAsync("abc.bin", new RandomStream(3));
using (var anotherClient = new RavenFileSystemClient(GetServerUrl(false, server.SystemDatabase.ServerUrl), "ShouldThrow_ApiKeyDoesnContainsThisFS", apiKey: apiKey))
{
await anotherClient.EnsureFileSystemExistsAsync(); // will pass because by using this api key we have access to <system> database
ErrorResponseException errorResponse = null;
try
{
await anotherClient.UploadAsync("def.bin", new RandomStream(1)); // should throw because a file system ShouldThrow_ApiKeyDoesnContainsThisFS isn't added to ApiKeyDefinition
}
catch (InvalidOperationException ex)
{
errorResponse = ex.InnerException as ErrorResponseException;
}
Assert.NotNull(errorResponse);
Assert.Equal(HttpStatusCode.Forbidden, errorResponse.StatusCode);
}
}
private static void ZeroTimeoutTest(RavenFileSystemClient destinationClient, Action action)
{
destinationClient.Config.SetConfig(RavenFileNameHelper.SyncLockNameForFile("test.bin"),
SynchronizationConfig(DateTime.MinValue)).Wait();
destinationClient.Config.SetConfig(SynchronizationConstants.RavenSynchronizationLockTimeout, new NameValueCollection
{
{
"value",
"\"00:00:00\""
}
}).Wait();
Assert.DoesNotThrow(() => action());
}
public ConflictNotificationTests()
{
destinationClient = NewClient(0);
sourceClient = NewClient(1);
}
private void UploadFilesSynchronously(out RavenFileSystemClient sourceClient,
out RavenFileSystemClient destinationClient, string fileName = "test.bin")
{
sourceClient = NewClient(1);
destinationClient = NewClient(0);
var sourceContent = new RandomlyModifiedStream(new RandomStream(10, 1), 0.01);
var destinationContent = new RandomlyModifiedStream(new RandomStream(10, 1), 0.01);
destinationClient.UploadAsync(fileName, EmptyData, destinationContent).Wait();
sourceClient.UploadAsync(fileName, EmptyData, sourceContent).Wait();
}
public SynchronizationNotificationTests()
{
destination = NewClient(0);
source = NewClient(1);
}
/// <summary>
/// Updates the replication information if needed.
/// </summary>
/// <param name="serverClient">The server client.</param>
#if SILVERLIGHT || NETFX_CORE
public Task UpdateReplicationInformationIfNeeded(RavenFileSystemClient serverClient)
public async Task Should_delete_file_on_all_destinations()
{
StartServerInstance(AddtitionalServerInstancePortNumber);
var sourceClient = NewClient(0);
var destination1Client = NewClient(1);
var destination2Client = new RavenFileSystemClient(ServerAddress(AddtitionalServerInstancePortNumber));
// upload file to first server and synchronize to others
await sourceClient.UploadAsync("test.bin", new RandomStream(10));
await sourceClient.Synchronization.StartAsync("test.bin", destination1Client.ServerUrl);
await sourceClient.Synchronization.StartAsync("test.bin", destination2Client.ServerUrl);
// delete file on source
await sourceClient.DeleteAsync("test.bin");
// set up destinations
await sourceClient.Config.SetConfig(SynchronizationConstants.RavenSynchronizationDestinations, new NameValueCollection
{
{
"url", destination1Client.ServerUrl
},
{
"url", destination2Client.ServerUrl
}
});
var destinationSyncResults = await sourceClient.Synchronization.SynchronizeDestinationsAsync();
foreach (var destinationSyncResult in destinationSyncResults)
{
foreach (var report in destinationSyncResult.Reports)
{
Assert.Null(report.Exception);
Assert.Equal(SynchronizationType.Delete, report.Type);
}
}
Assert.Null(await destination1Client.GetMetadataForAsync("test.bin"));
Assert.Null(await destination1Client.GetMetadataForAsync("test.bin"));
}
public void Should_synchronize_to_all_destinations()
{
StartServerInstance(AddtitionalServerInstancePortNumber);
var sourceContent = SyncTestUtils.PrepareSourceStream(10000);
sourceContent.Position = 0;
var sourceClient = NewClient(0);
var destination1Client = NewClient(1);
var destination2Client = new RavenFileSystemClient(ServerAddress(AddtitionalServerInstancePortNumber));
var destination1Content = new RandomlyModifiedStream(sourceContent, 0.01);
sourceContent.Position = 0;
var destination2Content = new RandomlyModifiedStream(sourceContent, 0.01);
sourceContent.Position = 0;
destination1Client.UploadAsync("test.bin", destination1Content).Wait();
destination2Client.UploadAsync("test.bin", destination2Content).Wait();
sourceContent.Position = 0;
sourceClient.UploadAsync("test.bin", sourceContent).Wait();
sourceContent.Position = 0;
sourceClient.Config.SetConfig(SynchronizationConstants.RavenSynchronizationDestinations, new NameValueCollection
{
{
"url",
destination1Client
.ServerUrl
},
{
"url",
destination2Client
.ServerUrl
}
}).Wait();
var destinationSyncResults = sourceClient.Synchronization.SynchronizeDestinationsAsync().Result;
// we expect conflicts after first attempt of synchronization
Assert.Equal(2, destinationSyncResults.Length);
Assert.Equal("File test.bin is conflicted", destinationSyncResults[0].Reports.ToArray()[0].Exception.Message);
Assert.Equal("File test.bin is conflicted", destinationSyncResults[1].Reports.ToArray()[0].Exception.Message);
destination1Client.Synchronization.ResolveConflictAsync("test.bin", ConflictResolutionStrategy.RemoteVersion).Wait();
destination2Client.Synchronization.ResolveConflictAsync("test.bin", ConflictResolutionStrategy.RemoteVersion).Wait();
destinationSyncResults = sourceClient.Synchronization.SynchronizeDestinationsAsync().Result;
// check if reports match
Assert.Equal(2, destinationSyncResults.Length);
var result1 = destinationSyncResults[0].Reports.ToArray()[0];
Assert.Equal(sourceContent.Length, result1.BytesCopied + result1.BytesTransfered);
var result2 = destinationSyncResults[1].Reports.ToArray()[0];
Assert.Equal(sourceContent.Length, result2.BytesCopied + result2.BytesTransfered);
// check content of files
string destination1Md5;
using (var resultFileContent = new MemoryStream())
{
destination1Client.DownloadAsync("test.bin", resultFileContent).Wait();
resultFileContent.Position = 0;
destination1Md5 = resultFileContent.GetMD5Hash();
}
string destination2Md5;
using (var resultFileContent = new MemoryStream())
{
destination2Client.DownloadAsync("test.bin", resultFileContent).Wait();
resultFileContent.Position = 0;
destination2Md5 = resultFileContent.GetMD5Hash();
}
sourceContent.Position = 0;
var sourceMd5 = sourceContent.GetMD5Hash();
Assert.Equal(sourceMd5, destination1Md5);
Assert.Equal(sourceMd5, destination2Md5);
Assert.Equal(destination1Md5, destination2Md5);
}
public async Task Should_change_metadata_on_all_destinations()
{
StartServerInstance(AddtitionalServerInstancePortNumber);
var sourceClient = NewClient(0);
var destination1Client = NewClient(1);
var destination2Client = new RavenFileSystemClient(ServerAddress(AddtitionalServerInstancePortNumber));
var sourceContent = new MemoryStream();
var streamWriter = new StreamWriter(sourceContent);
var expected = new string('a', 1024*1024*10);
streamWriter.Write(expected);
streamWriter.Flush();
sourceContent.Position = 0;
await sourceClient.UploadAsync("test.txt", sourceContent);
await
sourceClient.Config.SetConfig(SynchronizationConstants.RavenSynchronizationDestinations, new NameValueCollection
{
{
"url", destination1Client.ServerUrl
},
{
"url", destination2Client.ServerUrl
}
});
// push file to all destinations
await sourceClient.Synchronization.SynchronizeDestinationsAsync();
// prevent pushing files after metadata update
await sourceClient.Config.DeleteConfig(SynchronizationConstants.RavenSynchronizationDestinations);
await sourceClient.UpdateMetadataAsync("test.txt", new NameValueCollection {{"value", "shouldBeSynchronized"}});
// add destinations again
await
sourceClient.Config.SetConfig(SynchronizationConstants.RavenSynchronizationDestinations, new NameValueCollection
{
{
"url", destination1Client.ServerUrl
},
{
"url", destination2Client.ServerUrl
}
});
// should synchronize metadata
var destinationSyncResults = sourceClient.Synchronization.SynchronizeDestinationsAsync().Result;
foreach (var destinationSyncResult in destinationSyncResults)
{
foreach (var report in destinationSyncResult.Reports)
{
Assert.Null(report.Exception);
Assert.Equal(SynchronizationType.MetadataUpdate, report.Type);
}
}
Assert.Equal("shouldBeSynchronized", destination1Client.GetMetadataForAsync("test.txt").Result["value"]);
Assert.Equal("shouldBeSynchronized", destination2Client.GetMetadataForAsync("test.txt").Result["value"]);
}
public static void TurnOffSynchronization(RavenFileSystemClient source)
{
source.Config.DeleteConfig(SynchronizationConstants.RavenSynchronizationDestinations).Wait();
}
public RemoteSignaturePartialAccess(RavenFileSystemClient.SynchronizationClient synchronizationClient, string fileName)
{
this.synchronizationClient = synchronizationClient;
_fileName = fileName;
}
public SynchronizationNotificationTests()
{
destination = NewClient(0);
source = NewClient(1);
}
private async Task <DestinationSyncResult> SynchronizeDestinationAsync(SynchronizationDestination destination,
bool forceSyncingContinuation)
{
try
{
ICredentials credentials = null;
if (string.IsNullOrEmpty(destination.Username) == false)
{
credentials = string.IsNullOrEmpty(destination.Domain)
? new NetworkCredential(destination.Username, destination.Password)
: new NetworkCredential(destination.Username, destination.Password, destination.Domain);
}
var destinationClient = new RavenFileSystemClient(destination.ServerUrl, destination.FileSystem,
apiKey: destination.ApiKey, credentials: credentials).Synchronization;
var lastETag = await destinationClient.GetLastSynchronizationFromAsync(storage.Id);
var activeTasks = synchronizationQueue.Active.ToList();
var filesNeedConfirmation = GetSyncingConfigurations(destination).Where(sync => activeTasks.All(x => x.FileName != sync.FileName)).ToList();
var confirmations = await ConfirmPushedFiles(filesNeedConfirmation, destinationClient);
var needSyncingAgain = new List <FileHeader>();
foreach (var confirmation in confirmations)
{
if (confirmation.Status == FileStatus.Safe)
{
Log.Debug("Destination server {0} said that file '{1}' is safe", destination, confirmation.FileName);
RemoveSyncingConfiguration(confirmation.FileName, destination.FileSystemUrl);
}
else
{
storage.Batch(accessor =>
{
var fileHeader = accessor.ReadFile(confirmation.FileName);
if (fileHeader != null)
{
needSyncingAgain.Add(fileHeader);
Log.Debug("Destination server {0} said that file '{1}' is {2}.",
destination, confirmation.FileName, confirmation.Status);
}
});
}
}
await EnqueueMissingUpdatesAsync(destinationClient, lastETag, needSyncingAgain);
var reports = await Task.WhenAll(SynchronizePendingFilesAsync(destinationClient, forceSyncingContinuation));
var destinationSyncResult = new DestinationSyncResult
{
DestinationServer = destination.ServerUrl,
DestinationFileSystem = destination.FileSystem
};
if (reports.Length > 0)
{
var successfulSynchronizationsCount = reports.Count(x => x.Exception == null);
var failedSynchronizationsCount = reports.Count(x => x.Exception != null);
if (successfulSynchronizationsCount > 0 || failedSynchronizationsCount > 0)
{
Log.Debug(
"Synchronization to a destination {0} has completed. {1} file(s) were synchronized successfully, {2} synchronization(s) were failed",
destination.FileSystemUrl, successfulSynchronizationsCount, failedSynchronizationsCount);
}
destinationSyncResult.Reports = reports;
}
return(destinationSyncResult);
}
catch (Exception ex)
{
Log.WarnException(string.Format("Failed to perform a synchronization to a destination {0}", destination), ex);
return(new DestinationSyncResult
{
DestinationServer = destination.ServerUrl,
DestinationFileSystem = destination.FileSystem,
Exception = ex
});
}
}
public Notifications()
{
client = NewClient();
}
