public static void TurnOnSynchronization(RavenFileSystemClient source, RavenFileSystemClient destination) { source.Config.SetConfig(SynchronizationConstants.RavenSynchronizationDestinations, new NameValueCollection { {"url", destination.ServerUrl}, }).Wait(); }
public RemoteRdcManager(RavenFileSystemClient ravenFileSystemClient, ISignatureRepository localSignatureRepository, ISignatureRepository remoteCacheSignatureRepository) { this.localSignatureRepository = localSignatureRepository; this.remoteCacheSignatureRepository = remoteCacheSignatureRepository; this.ravenFileSystemClient = ravenFileSystemClient; }
public static SynchronizationDestination ToSynchronizationDestination(this RavenFileSystemClient self) { var result = new SynchronizationDestination() { FileSystem = self.FileSystemName, ServerUrl = self.ServerUrl, ApiKey = self.ApiKey }; if (self.Credentials != null) { var networkCredential = self.Credentials as NetworkCredential; if (networkCredential != null) { result.Username = networkCredential.UserName; result.Password = networkCredential.Password; result.Domain = networkCredential.Domain; } else { throw new InvalidOperationException("Expected NetworkCredential object while get: " + self.Credentials); } } return(result); }
public override Task<SynchronizationReport> PerformAsync(RavenFileSystemClient.SynchronizationClient destination) { FileAndPages fileAndPages = null; Storage.Batch(accessor => fileAndPages = accessor.GetFile(FileName, 0, 0)); return destination.DeleteAsync(FileName, fileAndPages.Metadata, ServerInfo); }
public async Task CanCreateAndDeleteFileSystem() { var client = NewClient(); var adminClient = client.Admin; const string newFileSystemName = "testName_CanDeleteFileSystem"; await adminClient.CreateOrUpdateFileSystemAsync(new DatabaseDocument { Id = "Raven/FileSystem/" + newFileSystemName, Settings = { { "Raven/FileSystem/DataDir", Path.Combine("~", Path.Combine("FileSystems", newFileSystemName)) } } }, newFileSystemName); using (var createdFsClient = new RavenFileSystemClient(client.ServerUrl, newFileSystemName)) { await createdFsClient.UploadAsync("foo", new MemoryStream(new byte[] { 1 })); } var names = await adminClient.GetFileSystemsNames(); Assert.Contains(newFileSystemName, names); var stats = await adminClient.GetFileSystemsStats(); Assert.NotNull(stats.FirstOrDefault(x => x.Name == newFileSystemName)); await adminClient.DeleteFileSystemAsync(newFileSystemName); names = await adminClient.GetFileSystemsNames(); Assert.DoesNotContain(newFileSystemName, names); }
public async Task AdminClientWorkWithOAuthEnabled() { var client = NewClient(enableAuthentication: true, apiKey: apiKey); var adminClient = client.Admin; await adminClient.CreateFileSystemAsync(new DatabaseDocument { Id = "Raven/FileSystem/" + "testName", Settings = { { "Raven/FileSystem/DataDir", Path.Combine("~", Path.Combine("FileSystems", "testName")) } } }, "testName"); var names = await adminClient.GetFileSystemsNames(); Assert.Equal(1, names.Length); // will not return 'testName' file system name because used apiKey doesn't have access to a such file system Assert.Equal("AdminClientWorkWithOAuthEnabled", names[0]); var stats = await adminClient.GetFileSystemsStats(); Assert.Equal(0, stats.Count); // 0 because our fs aren't active using (var createdFsClient = new RavenFileSystemClient(client.ServerUrl, "testName")) { await createdFsClient.UploadAsync("foo", new MemoryStream(new byte[] { 1 })); } await adminClient.DeleteFileSystemAsync("testName", true); }
public async Task ShouldThrowWhenUsedApiKeyDefinitionDoesNotContainFileSystem() { var client = NewClient(enableAuthentication: true, apiKey: apiKey); var server = GetServer(); await client.UploadAsync("abc.bin", new RandomStream(3)); using (var anotherClient = new RavenFileSystemClient(GetServerUrl(false, server.SystemDatabase.ServerUrl), "ShouldThrow_ApiKeyDoesnContainsThisFS", apiKey: apiKey)) { await anotherClient.EnsureFileSystemExistsAsync(); // will pass because by using this api key we have access to <system> database ErrorResponseException errorResponse = null; try { await anotherClient.UploadAsync("def.bin", new RandomStream(1)); // should throw because a file system ShouldThrow_ApiKeyDoesnContainsThisFS isn't added to ApiKeyDefinition } catch (InvalidOperationException ex) { errorResponse = ex.InnerException as ErrorResponseException; } Assert.NotNull(errorResponse); Assert.Equal(HttpStatusCode.Forbidden, errorResponse.StatusCode); } }
public async Task AdminClientWorkWithWinAuthEnabled() { var client = NewClient(enableAuthentication: true, credentials: new NetworkCredential(username, password, domain)); var adminClient = client.Admin; await adminClient.CreateFileSystemAsync(new DatabaseDocument { Id = "Raven/FileSystem/" + "testName", Settings = { { "Raven/FileSystem/DataDir", Path.Combine("~", Path.Combine("FileSystems", "testName")) } } }, "testName"); using (var createdFsClient = new RavenFileSystemClient(client.ServerUrl, "testName", new NetworkCredential(username, password, domain))) { await createdFsClient.UploadAsync("foo", new MemoryStream(new byte[] { 1 })); } var names = await adminClient.GetFileSystemsNames(); Assert.Contains("testName", names); var stats = await adminClient.GetFileSystemsStats(); Assert.NotNull(stats.FirstOrDefault(x => x.Name == "testName")); await adminClient.DeleteFileSystemAsync("testName"); names = await adminClient.GetFileSystemsNames(); Assert.DoesNotContain("testName", names); }
/// <summary> /// Refreshes the replication information. /// Expert use only. /// </summary> public async Task RefreshReplicationInformationAsync(RavenFileSystemClient serverClient) { var serverHash = ServerHash.GetServerHash(serverClient.ServerUrl); try { var result = await serverClient.Config.GetConfig(SynchronizationConstants.RavenSynchronizationDestinations); if (result == null) { LastReplicationUpdate = SystemTime.UtcNow; // checked and not found } else { var urls = result.GetValues("url"); replicationDestinations = urls == null ? new List <string>() : urls.ToList(); } } catch (Exception e) { log.ErrorException("Could not contact master for new replication information", e); replicationDestinations = ReplicationInformerLocalCache.TryLoadReplicationInformationFromLocalCache(serverHash).ToList(); LastReplicationUpdate = SystemTime.UtcNow; return; } failureCounts[serverClient.ServerUrl] = new FailureCounter(); // we just hit the master, so we can reset its failure count ReplicationInformerLocalCache.TrySavingReplicationInformationToLocalCache(serverHash, replicationDestinations); UpdateReplicationInformationFromDocument(replicationDestinations); LastReplicationUpdate = SystemTime.UtcNow; }
public async Task ShouldThrowWhenWindowsDocumentDoesNotContainFileSystem() { // in this test be careful if the specified credentials belong to admin user or not var client = NewClient(enableAuthentication: true, credentials: new NetworkCredential(username, password, domain)); var server = GetServer(); await client.UploadAsync("abc.bin", new RandomStream(3)); using (var anotherClient = new RavenFileSystemClient(GetServerUrl(false, server.SystemDatabase.ServerUrl), "ShouldThrow_WindowsDocumentDoesnContainsThisFS", credentials: new NetworkCredential(username, password, domain))) { await anotherClient.EnsureFileSystemExistsAsync(); // will pass because by using this api key we have access to <system> database ErrorResponseException errorResponse = null; try { await anotherClient.UploadAsync("def.bin", new RandomStream(1)); // should throw because a file system ShouldThrow_ApiKeyDoesnContainsThisFS isn't added to ApiKeyDefinition } catch (InvalidOperationException ex) { errorResponse = ex.InnerException as ErrorResponseException; } Assert.NotNull(errorResponse); Assert.Equal(HttpStatusCode.Forbidden, errorResponse.StatusCode); } }
public async Task WillUseDefaultNetworkCredentialsWhenServerRequiresAuthentication() { var server = CreateRavenDbServer(Ports[0], fileSystemName: "WillUseDefaultCredentials", enableAuthentication: true); // enable authentication using (var client = new RavenFileSystemClient(GetServerUrl(false, server.SystemDatabase.ServerUrl), "WillUseDefaultCredentials")) { await client.Admin.CreateFileSystemAsync(new DatabaseDocument() { Id = "Raven/FileSystem/" + client.FileSystemName, Settings = { { "Raven/FileSystem/DataDir", Path.Combine("FileSystems", client.FileSystemName) } } }); await client.UploadAsync("a", new MemoryStream(new byte[] { 1, 2 })); var ms = new MemoryStream(); await client.DownloadAsync("a", ms); var array = ms.ToArray(); Assert.Equal(1, array[0]); Assert.Equal(2, array[1]); } }
public async Task Execute(string[] args) { var documentStore = new DocumentStore { Url = uri.ToString(), } .Initialize(); var client = new RavenFileSystemClient(uri.ToString(), filesystem); Console.WriteLine("=== Available File Systems ==="); bool doesFileSystemExists = false; foreach (string name in await client.Admin.GetFileSystemsNames()) { Console.WriteLine(name); doesFileSystemExists |= name.ToLower() == filesystem; } if (!doesFileSystemExists) { var filesystemDocument = new DatabaseDocument() { Id = "Raven/FileSystems/" + filesystem }; filesystemDocument.Settings["Raven/FileSystem/DataDir"] = Path.Combine(filesystemLocation, filesystem); await client.Admin.CreateFileSystemAsync(filesystemDocument, filesystem); } Console.WriteLine(); foreach ( var file in directory.GetFiles()) { await client.UploadAsync(file.Name, file.OpenRead()); } }
static void DownloadFiles(RavenFileSystemClient client) { var files = Directory.GetFiles(@"C:\@DemoData"); Console.WriteLine("Downloading {0} files", files.Length); foreach (var f in files) { Console.WriteLine(); var filename = Path.GetFileName(f); try { var sw = new Stopwatch(); sw.Start(); var stream = File.Create(Path.Combine(@"C:\@DemoFiles", filename)); Console.WriteLine("** Downloading: {0}", filename); client.DownloadAsync(filename, stream).Wait(); stream.Close(); stream.Dispose(); sw.Stop(); Console.WriteLine("** Done: {0}: Seconds {1}", filename, (sw.ElapsedMilliseconds / 1000)); } catch (Exception ex) { Console.WriteLine(ex.InnerException); } } }
static void UploadFiles(RavenFileSystemClient client) { var files = Directory.GetFiles(@"C:\@DemoData"); Console.WriteLine("Uploading {0} files", files.Length); foreach (var f in files) { Console.WriteLine(); var filename = Path.GetFileName(f); var fileInfo = new System.IO.FileInfo(f); try { var sw = new Stopwatch(); sw.Start(); Console.WriteLine("** Uploading: {0}, Size: {1} KB", filename, (fileInfo.Length / 1024)); client.UploadAsync(filename, new NameValueCollection(), File.OpenRead(f)).Wait(); sw.Stop(); Console.WriteLine("** Done: {0}: Seconds {1}", filename, (sw.ElapsedMilliseconds / 1000)); } catch (Exception ex) { Console.WriteLine(ex.InnerException); } } }
public async Task WillUseDefaultNetworkCredentialsWhenServerRequiresAuthentication() { var server = CreateRavenDbServer(Ports[0], fileSystemName: "WillUseDefaultCredentials", enableAuthentication: true); // enable authentication using (var client = new RavenFileSystemClient(GetServerUrl(false, server.SystemDatabase.ServerUrl), "WillUseDefaultCredentials")) { await client.Admin.CreateFileSystemAsync(new DatabaseDocument() { Id = "Raven/FileSystem/" + client.FileSystemName, Settings = { {"Raven/FileSystem/DataDir", Path.Combine("~", Path.Combine("FileSystems", client.FileSystemName))} } }); await client.UploadAsync("a", new NameValueCollection(), new MemoryStream(new byte[] {1, 2})); var ms = new MemoryStream(); await client.DownloadAsync("a", ms); var array = ms.ToArray(); Assert.Equal(1, array[0]); Assert.Equal(2, array[1]); } }
public async Task Can_get_stats_for_all_active_file_systems() { var client = NewClient(); var server = GetServer(); using (var anotherClient = new RavenFileSystemClient(GetServerUrl(false, server.SystemDatabase.ServerUrl), "test")) { await anotherClient.EnsureFileSystemExistsAsync(); await client.UploadAsync("test1", new RandomStream(10)); // will make it active await anotherClient.UploadAsync("test1", new RandomStream(10)); // will make it active await client.UploadAsync("test2", new RandomStream(10)); var stats = await anotherClient.Admin.GetFileSystemsStats(); var stats1 = stats.FirstOrDefault(x => x.Name == client.FileSystemName); Assert.NotNull(stats1); var stats2 = stats.FirstOrDefault(x => x.Name == anotherClient.FileSystemName); Assert.NotNull(stats2); Assert.Equal(2, stats1.Metrics.Requests.Count); Assert.Equal(1, stats2.Metrics.Requests.Count); Assert.Equal(0, stats1.ActiveSyncs.Count); Assert.Equal(0, stats1.PendingSyncs.Count); Assert.Equal(0, stats2.ActiveSyncs.Count); Assert.Equal(0, stats2.PendingSyncs.Count); } }
protected RavenFileSystemClient CreateFileSystemClient(RavenConnectionStringOptions options, string fileSystemName) { var fsClient = new RavenFileSystemClient(options.Url, fileSystemName, apiKey: options.ApiKey, credentials: options.Credentials); ValidateThatServerIsUpAndFileSystemExists(fsClient); return(fsClient); }
public ApplicationModel() { Notifications = new ObservableCollection<Notification>(); Notifications.CollectionChanged += delegate { OnPropertyChanged(() => ErrorCount); }; Client = new RavenFileSystemClient(DetermineUri()) {IsObservingFailedUploads = true}; AsyncOperations = new AsyncOperationsModel(); State = new ApplicationState(); }
public static void TurnOnSynchronization(RavenFileSystemClient source, params RavenFileSystemClient[] destinations) { source.Config.SetDestinationsConfig(destinations.Select(x => new SynchronizationDestination() { FileSystem = x.FileSystemName, ServerUrl = x.ServerUrl }).ToArray()).Wait(); }
public SynchronizationStorageTests() { source = NewClient(0); destination = NewClient(1); sourceRfs = GetRavenFileSystem(0); destinationRfs = GetRavenFileSystem(1); }
protected RavenFileSystemClient CreateFileSystemClient(RavenConnectionStringOptions options, string fileSystemName) { var fsClient = new RavenFileSystemClient(options.Url, fileSystemName, apiKey: options.ApiKey, credentials: options.Credentials); ValidateThatServerIsUpAndFileSystemExists(fsClient); return fsClient; }
private static void ZeroTimeoutTest(RavenFileSystemClient destinationClient, Action action) { destinationClient.Config.SetConfig(RavenFileNameHelper.SyncLockNameForFile("test.bin"), SynchronizationConfig(DateTime.MinValue)).Wait(); destinationClient.Config.SetConfig(SynchronizationConstants.RavenSynchronizationLockTimeout, TimeSpan.FromSeconds(0)).Wait(); Assert.DoesNotThrow(() => action()); }
public override async Task<SynchronizationReport> PerformAsync(string destination) { AssertLocalFileExistsAndIsNotConflicted(FileMetadata); var destinationRavenFileSystemClient = new RavenFileSystemClient(destination); var destinationMetadata = await destinationRavenFileSystemClient.GetMetadataForAsync(FileName); if (destinationMetadata == null) { // if file doesn't exist on destination server - upload it there return await UploadToAsync(destination); } var destinationServerRdcStats = await destinationRavenFileSystemClient.Synchronization.GetRdcStatsAsync(); if (!IsRemoteRdcCompatible(destinationServerRdcStats)) throw new SynchronizationException("Incompatible RDC version detected on destination server"); var conflict = CheckConflictWithDestination(FileMetadata, destinationMetadata, ServerInfo.Url); if (conflict != null) return await ApplyConflictOnDestinationAsync(conflict, destination, ServerInfo.Url, log); using (var localSignatureRepository = new StorageSignatureRepository(Storage, FileName)) using (var remoteSignatureCache = new VolatileSignatureRepository(FileName)) { var localRdcManager = new LocalRdcManager(localSignatureRepository, Storage, sigGenerator); var destinationRdcManager = new RemoteRdcManager(destinationRavenFileSystemClient, localSignatureRepository, remoteSignatureCache); log.Debug("Starting to retrieve signatures of a local file '{0}'.", FileName); Cts.Token.ThrowIfCancellationRequested(); // first we need to create a local file signatures before we synchronize with remote ones var localSignatureManifest = await localRdcManager.GetSignatureManifestAsync(FileDataInfo); log.Debug("Number of a local file '{0}' signatures was {1}.", FileName, localSignatureManifest.Signatures.Count); if (localSignatureManifest.Signatures.Count > 0) { var destinationSignatureManifest = await destinationRdcManager.SynchronizeSignaturesAsync(FileDataInfo, Cts.Token); if (destinationSignatureManifest.Signatures.Count > 0) { return await SynchronizeTo(destination, localSignatureRepository, remoteSignatureCache, localSignatureManifest, destinationSignatureManifest); } } return await UploadToAsync(destination); } }
public static SynchronizationReport ResolveConflictAndSynchronize(RavenFileSystemClient sourceClient, RavenFileSystemClient destinationClient, string fileName) { var shouldBeConflict = sourceClient.Synchronization.StartAsync(fileName, destinationClient).Result; Assert.NotNull(shouldBeConflict.Exception); destinationClient.Synchronization.ResolveConflictAsync(fileName, ConflictResolutionStrategy.RemoteVersion).Wait(); return(sourceClient.Synchronization.StartAsync(fileName, destinationClient).Result); }
public static SynchronizationReport ResolveConflictAndSynchronize(RavenFileSystemClient sourceClient, RavenFileSystemClient destinationClient, string fileName) { var shouldBeConflict = sourceClient.Synchronization.StartAsync(fileName, destinationClient).Result; Assert.NotNull(shouldBeConflict.Exception); destinationClient.Synchronization.ResolveConflictAsync(fileName, ConflictResolutionStrategy.RemoteVersion).Wait(); return sourceClient.Synchronization.StartAsync(fileName, destinationClient).Result; }
static void Main(string[] args) { //var fs = new RavenFileSystemClient("http://storage.wolfeautomation.com/"); // var fs = new RavenFileSystemClient("http://localfs/"); var fs = new RavenFileSystemClient("http://reduction:9090/"); SimpleTest(fs); // UploadFiles(fs); //DownloadFiles(fs); }
public override Task<SynchronizationReport> PerformAsync(RavenFileSystemClient.SynchronizationClient destination) { AssertLocalFileExistsAndIsNotConflicted(FileMetadata); var conflict = CheckConflictWithDestination(FileMetadata, destinationMetadata, ServerInfo.FileSystemUrl); if (conflict != null) return ApplyConflictOnDestinationAsync(conflict, destination, ServerInfo.FileSystemUrl, log); return destination.UpdateMetadataAsync(FileName, FileMetadata, ServerInfo); }
public SynchronizationMultipartRequest(RavenFileSystemClient.SynchronizationClient destination, ServerInfo serverInfo, string fileName, RavenJObject sourceMetadata, Stream sourceStream, IList<RdcNeed> needList) { this.destination = destination; this.serverInfo = serverInfo; this.fileName = fileName; this.sourceMetadata = sourceMetadata; this.sourceStream = sourceStream; this.needList = needList; syncingBoundary = "syncing"; }
private void UploadFilesSynchronously(out RavenFileSystemClient sourceClient, out RavenFileSystemClient destinationClient, string fileName = "test.bin") { sourceClient = NewClient(1); destinationClient = NewClient(0); var sourceContent = new RandomlyModifiedStream(new RandomStream(10, 1), 0.01); var destinationContent = new RandomlyModifiedStream(new RandomStream(10, 1), 0.01); destinationClient.UploadAsync(fileName, destinationContent).Wait(); sourceClient.UploadAsync(fileName, sourceContent).Wait(); }
public Task UpdateReplicationInformationIfNeeded(RavenFileSystemClient serverClient) #endif { if (Conventions.FailoverBehavior == FailoverBehavior.FailImmediately) { return(new CompletedTask()); } if (LastReplicationUpdate.AddMinutes(5) > SystemTime.UtcNow) { return(new CompletedTask()); } lock (replicationLock) { if (firstTime) { var serverHash = ServerHash.GetServerHash(serverClient.ServerUrl); var destinations = ReplicationInformerLocalCache.TryLoadReplicationInformationFromLocalCache(serverHash); if (destinations != null) { UpdateReplicationInformationFromDocument(destinations); } } firstTime = false; if (LastReplicationUpdate.AddMinutes(5) > SystemTime.UtcNow) { return(new CompletedTask()); } var taskCopy = refreshReplicationInformationTask; if (taskCopy != null) { return(taskCopy); } return(refreshReplicationInformationTask = RefreshReplicationInformationAsync(serverClient) .ContinueWith(task => { if (task.Exception != null) { log.ErrorException("Failed to refresh replication information", task.Exception); } refreshReplicationInformationTask = null; })); } }
protected virtual RavenFileSystemClient NewClient(int index = 0, bool fiddler = false, bool enableAuthentication = false, string apiKey = null, ICredentials credentials = null, string requestedStorage = null, [CallerMemberName] string fileSystemName = null) { fileSystemName = NormalizeFileSystemName(fileSystemName); var server = CreateRavenDbServer(Ports[index], fileSystemName: fileSystemName, enableAuthentication: enableAuthentication, requestedStorage: requestedStorage); var client = new RavenFileSystemClient(GetServerUrl(fiddler, server.SystemDatabase.ServerUrl), fileSystemName, apiKey: apiKey, credentials: credentials); client.EnsureFileSystemExistsAsync().Wait(); ravenFileSystemClients.Add(client); return(client); }
public async Task <SynchronizationReport> SynchronizeFileToAsync(string fileName, SynchronizationDestination destination) { ICredentials credentials = null; if (string.IsNullOrEmpty(destination.Username) == false) { credentials = string.IsNullOrEmpty(destination.Domain) ? new NetworkCredential(destination.Username, destination.Password) : new NetworkCredential(destination.Username, destination.Password, destination.Domain); } var destinationClient = new RavenFileSystemClient(destination.ServerUrl, destination.FileSystem, apiKey: destination.ApiKey, credentials: credentials).Synchronization; RavenJObject destinationMetadata; try { destinationMetadata = await destinationClient.GetMetadataForAsync(fileName); } catch (Exception ex) { var exceptionMessage = "Could not get metadata details for " + fileName + " from " + destination.FileSystemUrl; Log.WarnException(exceptionMessage, ex); return(new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown) { Exception = new SynchronizationException(exceptionMessage, ex) }); } RavenJObject localMetadata = GetLocalMetadata(fileName); NoSyncReason reason; SynchronizationWorkItem work = synchronizationStrategy.DetermineWork(fileName, localMetadata, destinationMetadata, FileSystemUrl, out reason); if (work == null) { Log.Debug("File '{0}' was not synchronized to {1}. {2}", fileName, destination.FileSystemUrl, reason.GetDescription()); return(new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown) { Exception = new SynchronizationException(reason.GetDescription()) }); } return(await PerformSynchronizationAsync(destinationClient, work)); }
public async Task File_rename_should_be_propagated() { StartServerInstance(AddtitionalServerInstancePortNumber); var content = new MemoryStream(new byte[] {1, 2, 3}); var server1 = NewClient(0); var server2 = NewClient(1); var server3 = new RavenFileSystemClient(ServerAddress(AddtitionalServerInstancePortNumber)); content.Position = 0; server1.UploadAsync("test.bin", new NameValueCollection {{"test", "value"}}, content).Wait(); SyncTestUtils.TurnOnSynchronization(server1, server2); Assert.Null(server1.Synchronization.SynchronizeDestinationsAsync().Result[0].Exception); SyncTestUtils.TurnOnSynchronization(server2, server3); Assert.Null(server2.Synchronization.SynchronizeDestinationsAsync().Result[0].Exception); SyncTestUtils.TurnOffSynchronization(server1); server1.RenameAsync("test.bin", "rename.bin").Wait(); SyncTestUtils.TurnOnSynchronization(server1, server2); var secondServer1Synchronization = await server1.Synchronization.SynchronizeDestinationsAsync(); Assert.Null(secondServer1Synchronization[0].Exception); Assert.Equal(SynchronizationType.Rename, secondServer1Synchronization[0].Reports.ToArray()[0].Type); var secondServer2Synchronization = await server2.Synchronization.SynchronizeDestinationsAsync(); Assert.Null(secondServer2Synchronization[0].Exception); Assert.Equal(SynchronizationType.Rename, secondServer2Synchronization[0].Reports.ToArray()[0].Type); // On all servers should be file named "rename.bin" Assert.Equal(1, server1.BrowseAsync().Result.Count()); Assert.Equal("rename.bin", server1.BrowseAsync().Result[0].Name); Assert.Equal(1, server2.BrowseAsync().Result.Count()); Assert.Equal("rename.bin", server2.BrowseAsync().Result[0].Name); Assert.Equal(1, server3.BrowseAsync().Result.Count()); Assert.Equal("rename.bin", server3.BrowseAsync().Result[0].Name); }
static void SimpleTest(RavenFileSystemClient fs) { var ms = new MemoryStream(); var streamWriter = new StreamWriter(ms); var expected = new string('a', 1024); streamWriter.Write(expected); streamWriter.Flush(); ms.Position = 0; fs.UploadAsync("abc.txt", ms).Wait(); var ms2 = new MemoryStream(); fs.DownloadAsync("abc.txt", ms2).Wait(); ms2.Position = 0; var actual = new StreamReader(ms2).ReadToEnd(); Console.WriteLine(actual.Length); }
public static async Task EnsureFileSystemExistsAsync(this RavenFileSystemClient client) { var existingSystems = await client.Admin.GetFileSystemsNames(); if (existingSystems.Any(x => x.Equals(client.FileSystemName, StringComparison.InvariantCultureIgnoreCase))) { return; } await client.Admin.CreateFileSystemAsync(new DatabaseDocument { Id = "Raven/FileSystem/" + client.FileSystemName, Settings = { { "Raven/FileSystem/DataDir", Path.Combine("FileSystems", client.FileSystemName) } } }); }
protected void ValidateThatServerIsUpAndFileSystemExists(RavenFileSystemClient fsClient) { var shouldDispose = false; try { var fileSystemStats = fsClient.StatsAsync().Result; } catch (Exception e) { shouldDispose = true; var responseException = e as ErrorResponseException; if (responseException != null && responseException.StatusCode == HttpStatusCode.ServiceUnavailable && responseException.Message.StartsWith("Could not find a file system named:")) { throw new InvalidOperationException( string.Format( "Migration tool does not support file system creation (file system '{0}' on server '{1}' must exist before running this tool).", fsClient.FileSystemName, fsClient.ServerUrl), e); } if (e.InnerException != null) { var webException = e.InnerException as WebException; if (webException != null) { throw new InvalidOperationException(string.Format("Migration tool encountered a connection problem: '{0}'.", webException.Message), webException); } } throw new InvalidOperationException(string.Format("Migration tool encountered a connection problem: '{0}'.", e.Message), e); } finally { if (shouldDispose) { fsClient.Dispose(); } } }
public async Task<SynchronizationReport> SynchronizeFileToAsync(string fileName, string destinationUrl) { var destinationClient = new RavenFileSystemClient(destinationUrl); NameValueCollection destinationMetadata; try { destinationMetadata = await destinationClient.GetMetadataForAsync(fileName); } catch (Exception ex) { var exceptionMessage = "Could not get metadata details for " + fileName + " from " + destinationUrl; Log.WarnException(exceptionMessage, ex); return new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown) { Exception = new SynchronizationException(exceptionMessage, ex) }; } NameValueCollection localMetadata = GetLocalMetadata(fileName); NoSyncReason reason; SynchronizationWorkItem work = synchronizationStrategy.DetermineWork(fileName, localMetadata, destinationMetadata, ServerUrl, out reason); if (work == null) { Log.Debug("File '{0}' was not synchronized to {1}. {2}", fileName, destinationUrl, reason.GetDescription()); return new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown) { Exception = new SynchronizationException(reason.GetDescription()) }; } return await PerformSynchronizationAsync(destinationClient.ServerUrl, work); }
public async Task Execute(string[] args) { var documentStore = new DocumentStore { Url = uri.ToString(), } .Initialize(); var client = new RavenFileSystemClient(uri.ToString(), filesystem); Console.WriteLine("=== Available File Systems ==="); bool doesFileSystemExists = false; foreach (string name in await client.Admin.GetFileSystemsNames()) { Console.WriteLine(name); doesFileSystemExists |= name.ToLower() == filesystem; } if (!doesFileSystemExists) { var filesystemDocument = new DatabaseDocument() { Id = "Raven/FileSystems/" + filesystem }; filesystemDocument.Settings["Raven/FileSystem/DataDir"] = Path.Combine(filesystemLocation, filesystem); await client.Admin.CreateFileSystemAsync(filesystemDocument, filesystem); } Console.WriteLine(); foreach (var file in directory.GetFiles()) { await client.UploadAsync(file.Name, file.OpenRead()); } }
public async Task<SynchronizationReport> SynchronizeFileToAsync(string fileName, SynchronizationDestination destination) { ICredentials credentials = null; if (string.IsNullOrEmpty(destination.Username) == false) { credentials = string.IsNullOrEmpty(destination.Domain) ? new NetworkCredential(destination.Username, destination.Password) : new NetworkCredential(destination.Username, destination.Password, destination.Domain); } var destinationClient = new RavenFileSystemClient(destination.ServerUrl, destination.FileSystem, apiKey: destination.ApiKey, credentials: credentials).Synchronization; RavenJObject destinationMetadata; try { destinationMetadata = await destinationClient.GetMetadataForAsync(fileName); } catch (Exception ex) { var exceptionMessage = "Could not get metadata details for " + fileName + " from " + destination.FileSystemUrl; Log.WarnException(exceptionMessage, ex); return new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown) { Exception = new SynchronizationException(exceptionMessage, ex) }; } RavenJObject localMetadata = GetLocalMetadata(fileName); NoSyncReason reason; SynchronizationWorkItem work = synchronizationStrategy.DetermineWork(fileName, localMetadata, destinationMetadata, FileSystemUrl, out reason); if (work == null) { Log.Debug("File '{0}' was not synchronized to {1}. {2}", fileName, destination.FileSystemUrl, reason.GetDescription()); return new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown) { Exception = new SynchronizationException(reason.GetDescription()) }; } return await PerformSynchronizationAsync(destinationClient, work); }
private async Task<DestinationSyncResult> SynchronizeDestinationAsync(SynchronizationDestination destination, bool forceSyncingContinuation) { try { ICredentials credentials = null; if (string.IsNullOrEmpty(destination.Username) == false) { credentials = string.IsNullOrEmpty(destination.Domain) ? new NetworkCredential(destination.Username, destination.Password) : new NetworkCredential(destination.Username, destination.Password, destination.Domain); } var destinationClient = new RavenFileSystemClient(destination.ServerUrl, destination.FileSystem, apiKey: destination.ApiKey, credentials: credentials).Synchronization; var lastETag = await destinationClient.GetLastSynchronizationFromAsync(storage.Id); var activeTasks = synchronizationQueue.Active.ToList(); var filesNeedConfirmation = GetSyncingConfigurations(destination).Where(sync => activeTasks.All(x => x.FileName != sync.FileName)).ToList(); var confirmations = await ConfirmPushedFiles(filesNeedConfirmation, destinationClient); var needSyncingAgain = new List<FileHeader>(); foreach (var confirmation in confirmations) { if (confirmation.Status == FileStatus.Safe) { Log.Debug("Destination server {0} said that file '{1}' is safe", destination, confirmation.FileName); RemoveSyncingConfiguration(confirmation.FileName, destination.FileSystemUrl); } else { storage.Batch(accessor => { var fileHeader = accessor.ReadFile(confirmation.FileName); if (fileHeader != null) { needSyncingAgain.Add(fileHeader); Log.Debug("Destination server {0} said that file '{1}' is {2}.", destination, confirmation.FileName, confirmation.Status); } }); } } await EnqueueMissingUpdatesAsync(destinationClient, lastETag, needSyncingAgain); var reports = await Task.WhenAll(SynchronizePendingFilesAsync(destinationClient, forceSyncingContinuation)); var destinationSyncResult = new DestinationSyncResult { DestinationServer = destination.ServerUrl, DestinationFileSystem = destination.FileSystem }; if (reports.Length > 0) { var successfulSynchronizationsCount = reports.Count(x => x.Exception == null); var failedSynchronizationsCount = reports.Count(x => x.Exception != null); if (successfulSynchronizationsCount > 0 || failedSynchronizationsCount > 0) { Log.Debug( "Synchronization to a destination {0} has completed. {1} file(s) were synchronized successfully, {2} synchronization(s) were failed", destination.FileSystemUrl, successfulSynchronizationsCount, failedSynchronizationsCount); } destinationSyncResult.Reports = reports; } return destinationSyncResult; } catch (Exception ex) { Log.WarnException(string.Format("Failed to perform a synchronization to a destination {0}", destination), ex); return new DestinationSyncResult { DestinationServer = destination.ServerUrl, DestinationFileSystem = destination.FileSystem, Exception = ex }; } }
public Notifications() { client = NewClient(); }
private static void ZeroTimeoutTest(RavenFileSystemClient destinationClient, Action action) { destinationClient.Config.SetConfig(RavenFileNameHelper.SyncLockNameForFile("test.bin"), SynchronizationConfig(DateTime.MinValue)).Wait(); destinationClient.Config.SetConfig(SynchronizationConstants.RavenSynchronizationLockTimeout, new NameValueCollection { { "value", "\"00:00:00\"" } }).Wait(); Assert.DoesNotThrow(() => action()); }
public ConflictNotificationTests() { destinationClient = NewClient(0); sourceClient = NewClient(1); }
private void UploadFilesSynchronously(out RavenFileSystemClient sourceClient, out RavenFileSystemClient destinationClient, string fileName = "test.bin") { sourceClient = NewClient(1); destinationClient = NewClient(0); var sourceContent = new RandomlyModifiedStream(new RandomStream(10, 1), 0.01); var destinationContent = new RandomlyModifiedStream(new RandomStream(10, 1), 0.01); destinationClient.UploadAsync(fileName, EmptyData, destinationContent).Wait(); sourceClient.UploadAsync(fileName, EmptyData, sourceContent).Wait(); }
public SynchronizationNotificationTests() { destination = NewClient(0); source = NewClient(1); }
/// <summary> /// Updates the replication information if needed. /// </summary> /// <param name="serverClient">The server client.</param> #if SILVERLIGHT || NETFX_CORE public Task UpdateReplicationInformationIfNeeded(RavenFileSystemClient serverClient)
public async Task Should_delete_file_on_all_destinations() { StartServerInstance(AddtitionalServerInstancePortNumber); var sourceClient = NewClient(0); var destination1Client = NewClient(1); var destination2Client = new RavenFileSystemClient(ServerAddress(AddtitionalServerInstancePortNumber)); // upload file to first server and synchronize to others await sourceClient.UploadAsync("test.bin", new RandomStream(10)); await sourceClient.Synchronization.StartAsync("test.bin", destination1Client.ServerUrl); await sourceClient.Synchronization.StartAsync("test.bin", destination2Client.ServerUrl); // delete file on source await sourceClient.DeleteAsync("test.bin"); // set up destinations await sourceClient.Config.SetConfig(SynchronizationConstants.RavenSynchronizationDestinations, new NameValueCollection { { "url", destination1Client.ServerUrl }, { "url", destination2Client.ServerUrl } }); var destinationSyncResults = await sourceClient.Synchronization.SynchronizeDestinationsAsync(); foreach (var destinationSyncResult in destinationSyncResults) { foreach (var report in destinationSyncResult.Reports) { Assert.Null(report.Exception); Assert.Equal(SynchronizationType.Delete, report.Type); } } Assert.Null(await destination1Client.GetMetadataForAsync("test.bin")); Assert.Null(await destination1Client.GetMetadataForAsync("test.bin")); }
public void Should_synchronize_to_all_destinations() { StartServerInstance(AddtitionalServerInstancePortNumber); var sourceContent = SyncTestUtils.PrepareSourceStream(10000); sourceContent.Position = 0; var sourceClient = NewClient(0); var destination1Client = NewClient(1); var destination2Client = new RavenFileSystemClient(ServerAddress(AddtitionalServerInstancePortNumber)); var destination1Content = new RandomlyModifiedStream(sourceContent, 0.01); sourceContent.Position = 0; var destination2Content = new RandomlyModifiedStream(sourceContent, 0.01); sourceContent.Position = 0; destination1Client.UploadAsync("test.bin", destination1Content).Wait(); destination2Client.UploadAsync("test.bin", destination2Content).Wait(); sourceContent.Position = 0; sourceClient.UploadAsync("test.bin", sourceContent).Wait(); sourceContent.Position = 0; sourceClient.Config.SetConfig(SynchronizationConstants.RavenSynchronizationDestinations, new NameValueCollection { { "url", destination1Client .ServerUrl }, { "url", destination2Client .ServerUrl } }).Wait(); var destinationSyncResults = sourceClient.Synchronization.SynchronizeDestinationsAsync().Result; // we expect conflicts after first attempt of synchronization Assert.Equal(2, destinationSyncResults.Length); Assert.Equal("File test.bin is conflicted", destinationSyncResults[0].Reports.ToArray()[0].Exception.Message); Assert.Equal("File test.bin is conflicted", destinationSyncResults[1].Reports.ToArray()[0].Exception.Message); destination1Client.Synchronization.ResolveConflictAsync("test.bin", ConflictResolutionStrategy.RemoteVersion).Wait(); destination2Client.Synchronization.ResolveConflictAsync("test.bin", ConflictResolutionStrategy.RemoteVersion).Wait(); destinationSyncResults = sourceClient.Synchronization.SynchronizeDestinationsAsync().Result; // check if reports match Assert.Equal(2, destinationSyncResults.Length); var result1 = destinationSyncResults[0].Reports.ToArray()[0]; Assert.Equal(sourceContent.Length, result1.BytesCopied + result1.BytesTransfered); var result2 = destinationSyncResults[1].Reports.ToArray()[0]; Assert.Equal(sourceContent.Length, result2.BytesCopied + result2.BytesTransfered); // check content of files string destination1Md5; using (var resultFileContent = new MemoryStream()) { destination1Client.DownloadAsync("test.bin", resultFileContent).Wait(); resultFileContent.Position = 0; destination1Md5 = resultFileContent.GetMD5Hash(); } string destination2Md5; using (var resultFileContent = new MemoryStream()) { destination2Client.DownloadAsync("test.bin", resultFileContent).Wait(); resultFileContent.Position = 0; destination2Md5 = resultFileContent.GetMD5Hash(); } sourceContent.Position = 0; var sourceMd5 = sourceContent.GetMD5Hash(); Assert.Equal(sourceMd5, destination1Md5); Assert.Equal(sourceMd5, destination2Md5); Assert.Equal(destination1Md5, destination2Md5); }
public async Task Should_change_metadata_on_all_destinations() { StartServerInstance(AddtitionalServerInstancePortNumber); var sourceClient = NewClient(0); var destination1Client = NewClient(1); var destination2Client = new RavenFileSystemClient(ServerAddress(AddtitionalServerInstancePortNumber)); var sourceContent = new MemoryStream(); var streamWriter = new StreamWriter(sourceContent); var expected = new string('a', 1024*1024*10); streamWriter.Write(expected); streamWriter.Flush(); sourceContent.Position = 0; await sourceClient.UploadAsync("test.txt", sourceContent); await sourceClient.Config.SetConfig(SynchronizationConstants.RavenSynchronizationDestinations, new NameValueCollection { { "url", destination1Client.ServerUrl }, { "url", destination2Client.ServerUrl } }); // push file to all destinations await sourceClient.Synchronization.SynchronizeDestinationsAsync(); // prevent pushing files after metadata update await sourceClient.Config.DeleteConfig(SynchronizationConstants.RavenSynchronizationDestinations); await sourceClient.UpdateMetadataAsync("test.txt", new NameValueCollection {{"value", "shouldBeSynchronized"}}); // add destinations again await sourceClient.Config.SetConfig(SynchronizationConstants.RavenSynchronizationDestinations, new NameValueCollection { { "url", destination1Client.ServerUrl }, { "url", destination2Client.ServerUrl } }); // should synchronize metadata var destinationSyncResults = sourceClient.Synchronization.SynchronizeDestinationsAsync().Result; foreach (var destinationSyncResult in destinationSyncResults) { foreach (var report in destinationSyncResult.Reports) { Assert.Null(report.Exception); Assert.Equal(SynchronizationType.MetadataUpdate, report.Type); } } Assert.Equal("shouldBeSynchronized", destination1Client.GetMetadataForAsync("test.txt").Result["value"]); Assert.Equal("shouldBeSynchronized", destination2Client.GetMetadataForAsync("test.txt").Result["value"]); }
public static void TurnOffSynchronization(RavenFileSystemClient source) { source.Config.DeleteConfig(SynchronizationConstants.RavenSynchronizationDestinations).Wait(); }
public RemoteSignaturePartialAccess(RavenFileSystemClient.SynchronizationClient synchronizationClient, string fileName) { this.synchronizationClient = synchronizationClient; _fileName = fileName; }
private async Task <DestinationSyncResult> SynchronizeDestinationAsync(SynchronizationDestination destination, bool forceSyncingContinuation) { try { ICredentials credentials = null; if (string.IsNullOrEmpty(destination.Username) == false) { credentials = string.IsNullOrEmpty(destination.Domain) ? new NetworkCredential(destination.Username, destination.Password) : new NetworkCredential(destination.Username, destination.Password, destination.Domain); } var destinationClient = new RavenFileSystemClient(destination.ServerUrl, destination.FileSystem, apiKey: destination.ApiKey, credentials: credentials).Synchronization; var lastETag = await destinationClient.GetLastSynchronizationFromAsync(storage.Id); var activeTasks = synchronizationQueue.Active.ToList(); var filesNeedConfirmation = GetSyncingConfigurations(destination).Where(sync => activeTasks.All(x => x.FileName != sync.FileName)).ToList(); var confirmations = await ConfirmPushedFiles(filesNeedConfirmation, destinationClient); var needSyncingAgain = new List <FileHeader>(); foreach (var confirmation in confirmations) { if (confirmation.Status == FileStatus.Safe) { Log.Debug("Destination server {0} said that file '{1}' is safe", destination, confirmation.FileName); RemoveSyncingConfiguration(confirmation.FileName, destination.FileSystemUrl); } else { storage.Batch(accessor => { var fileHeader = accessor.ReadFile(confirmation.FileName); if (fileHeader != null) { needSyncingAgain.Add(fileHeader); Log.Debug("Destination server {0} said that file '{1}' is {2}.", destination, confirmation.FileName, confirmation.Status); } }); } } await EnqueueMissingUpdatesAsync(destinationClient, lastETag, needSyncingAgain); var reports = await Task.WhenAll(SynchronizePendingFilesAsync(destinationClient, forceSyncingContinuation)); var destinationSyncResult = new DestinationSyncResult { DestinationServer = destination.ServerUrl, DestinationFileSystem = destination.FileSystem }; if (reports.Length > 0) { var successfulSynchronizationsCount = reports.Count(x => x.Exception == null); var failedSynchronizationsCount = reports.Count(x => x.Exception != null); if (successfulSynchronizationsCount > 0 || failedSynchronizationsCount > 0) { Log.Debug( "Synchronization to a destination {0} has completed. {1} file(s) were synchronized successfully, {2} synchronization(s) were failed", destination.FileSystemUrl, successfulSynchronizationsCount, failedSynchronizationsCount); } destinationSyncResult.Reports = reports; } return(destinationSyncResult); } catch (Exception ex) { Log.WarnException(string.Format("Failed to perform a synchronization to a destination {0}", destination), ex); return(new DestinationSyncResult { DestinationServer = destination.ServerUrl, DestinationFileSystem = destination.FileSystem, Exception = ex }); } }