public async Task RemoveEntryFromRaftLogTest() { var leader = await CreateRaftClusterAndGetLeader(3); var database = GetDatabaseName(); await CreateDatabaseInClusterInner(new DatabaseRecord(database), 3, leader.WebUrl, null); using (var store = new DocumentStore { Database = database, Urls = new[] { leader.WebUrl } }.Initialize()) { leader.ServerStore.Engine.StateMachine.Validator = new TestCommandValidator(); var documentDatabase = await leader.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database); var cmd = new RachisConsensusTestBase.TestCommandWithRaftId("test", RaftIdGenerator.NewId()); _ = leader.ServerStore.Engine.CurrentLeader.PutAsync(cmd, new TimeSpan(2000)); var cmd2 = new CreateDatabaseOperation.CreateDatabaseCommand(new DatabaseRecord("Toli"), 1); _ = leader.ServerStore.SendToLeaderAsync(new AddDatabaseCommand(Guid.NewGuid().ToString()) { Record = new DatabaseRecord("Toli") { Topology = new DatabaseTopology { Members = new List <string> { "A", "B", "C" }, Rehabs = new List <string> { }, ReplicationFactor = 3 } }, Name = "Toli" }); foreach (var server in Servers) { Assert.False(leader.ServerStore.DatabasesLandlord.IsDatabaseLoaded("Toli")); } foreach (var server in Servers) { long index = 0; documentDatabase = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database); using (documentDatabase.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext context)) using (var tx = context.OpenReadTransaction()) { server.ServerStore.Engine.GetLastCommitIndex(context, out index, out long term); } server.ServerStore.Engine.RemoveEntryFromRaftLog(index + 1); } foreach (var server in Servers) { var val = WaitForValueAsync(() => server.ServerStore.DatabasesLandlord.IsDatabaseLoaded("Toli"), true); Assert.True(val.Result); } } }
private async Task AddDatabases() { await _locker.WaitAsync(); List <string> databases = null; List <string> missingDatabases = null; try { using (_server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) { context.OpenReadTransaction(); databases = _server .ServerStore .Cluster .ItemKeysStartingWith(context, Constants.Documents.Prefix, 0, long.MaxValue) .Select(x => x.Substring(Constants.Documents.Prefix.Length)) .ToList(); if (databases.Count == 0) { return; } var mapping = GetMapping(_server.ServerStore, context); missingDatabases = new List <string>(); foreach (var database in databases) { if (mapping.ContainsKey(database) == false) { missingDatabases.Add(database); } } if (missingDatabases.Count > 0) { context.CloseTransaction(); try { var result = await _server.ServerStore.SendToLeaderAsync(new UpdateSnmpDatabasesMappingCommand(missingDatabases, RaftIdGenerator.NewId())); await _server.ServerStore.Cluster.WaitForIndexNotification(result.Index); } catch (Exception e) when(e is TimeoutException || e is OperationCanceledException) { // we will update it in the OnDatabaseLoaded event return; } context.OpenReadTransaction(); mapping = GetMapping(_server.ServerStore, context); } foreach (var database in databases) { LoadDatabase(database, mapping[database]); } } } catch (Exception e) { var msg = "Failed to update the SNMP mapping"; if (databases?.Count > 0) { msg += $" databases to update: {string.Join(", ", databases)}"; } if (missingDatabases?.Count > 0) { msg += $" missing databases: {string.Join(", ", missingDatabases)}"; } if (Logger.IsOperationsEnabled) { Logger.Operations(msg, e); } } finally { _locker.Release(); } }
public static void SaveBackupStatus(PeriodicBackupStatus status, DocumentDatabase documentDatabase, Logger logger) { try { var command = new UpdatePeriodicBackupStatusCommand(documentDatabase.Name, RaftIdGenerator.NewId()) { PeriodicBackupStatus = status }; var result = AsyncHelpers.RunSync(() => documentDatabase.ServerStore.SendToLeaderAsync(command)); AsyncHelpers.RunSync(() => documentDatabase.ServerStore.WaitForCommitIndexChange(RachisConsensus.CommitIndexModification.GreaterOrEqual, result.Index)); if (logger.IsInfoEnabled) { logger.Info($"Periodic backup status with task id {status.TaskId} was updated"); } } catch (Exception e) { const string message = "Error saving the periodic backup status"; if (logger.IsOperationsEnabled) { logger.Operations(message, e); } } }
public async Task RemoveEntryFromRaftLogEP() { var(_, leader) = await CreateRaftCluster(3); var database = GetDatabaseName(); await CreateDatabaseInClusterInner(new DatabaseRecord(database), 3, leader.WebUrl, null); using (var store = new DocumentStore { Database = database, Urls = new[] { leader.WebUrl } }.Initialize()) { leader.ServerStore.Engine.StateMachine.Validator = new TestCommandValidator(); var documentDatabase = await leader.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database); var cmd = new RachisConsensusTestBase.TestCommandWithRaftId("test", RaftIdGenerator.NewId()); _ = leader.ServerStore.Engine.CurrentLeader.PutAsync(cmd, new TimeSpan(2000)); var cmd2 = new CreateDatabaseOperation.CreateDatabaseCommand(new DatabaseRecord("Toli"), 1); _ = leader.ServerStore.SendToLeaderAsync(new AddDatabaseCommand(Guid.NewGuid().ToString()) { Record = new DatabaseRecord("Toli") { Topology = new DatabaseTopology { Members = new List <string> { "A", "B", "C" }, Rehabs = new List <string> { }, ReplicationFactor = 3 } }, Name = "Toli" }); foreach (var server in Servers) { Assert.False(server.ServerStore.DatabasesLandlord.IsDatabaseLoaded("Toli")); } long index; using (documentDatabase.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext context)) using (var tx = context.OpenReadTransaction()) { leader.ServerStore.Engine.GetLastCommitIndex(context, out index, out long term); } List <string> nodelist = new List <string>(); var res = await WaitForValueAsync(async() => { nodelist = await store.Maintenance.SendAsync(new RemoveEntryFromRaftLogOperation(index + 1)); return(nodelist.Count); }, 3); Assert.Equal(3, res); long index2 = 0; foreach (var server in Servers) { await WaitForValueAsync(async() => { documentDatabase = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database); using (documentDatabase.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext context)) using (var tx = context.OpenReadTransaction()) { server.ServerStore.Engine.GetLastCommitIndex(context, out index2, out long term); } return(index2 > index); }, true); } Assert.True(index2 > index, $"State machine is stuck. raft index was {index}, after remove raft entry index is {index2} "); foreach (var server in Servers) { var val = WaitForValueAsync(() => server.ServerStore.DatabasesLandlord.IsDatabaseLoaded("Toli"), true); Assert.True(val.Result); Assert.Contains(server.ServerStore.NodeTag, nodelist); } } }
private async Task CreateDatabaseIfNeeded(string databaseName) { using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) using (context.OpenReadTransaction()) using (var rawRecord = _serverStore.Cluster.ReadRawDatabaseRecord(context, databaseName)) { if (rawRecord != null) { if (rawRecord.GetTopology().AllNodes.Contains(_serverStore.NodeTag) == false) { throw new InvalidOperationException( "Cannot migrate database because " + $"database doesn't exist on the current node ({_serverStore.NodeTag})"); } // database already exists return; } } var databaseRecord = new DatabaseRecord(databaseName) { Topology = new DatabaseTopology { Members = { _serverStore.NodeTag } } }; var(index, _) = await _serverStore.WriteDatabaseRecordAsync(databaseName, databaseRecord, null, RaftIdGenerator.NewId()); await _serverStore.Cluster.WaitForIndexNotification(index); }
private async Task AddIndexesFromDatabase(DocumentDatabase database) { var indexes = database.IndexStore.GetIndexes().ToList(); if (indexes.Count == 0) { return; } using (database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) { context.OpenReadTransaction(); var mapping = GetIndexMapping(context, database.ServerStore, database.Name); var missingIndexes = new List <string>(); foreach (var index in indexes) { if (mapping.ContainsKey(index.Name) == false) { missingIndexes.Add(index.Name); } } if (missingIndexes.Count > 0) { context.CloseTransaction(); var result = await database.ServerStore.SendToLeaderAsync(new UpdateSnmpDatabaseIndexesMappingCommand(database.Name, missingIndexes, RaftIdGenerator.NewId())); await database.ServerStore.Cluster.WaitForIndexNotification(result.Index); context.OpenReadTransaction(); mapping = GetIndexMapping(context, database.ServerStore, database.Name); } foreach (var index in indexes) { LoadIndex(index.Name, (int)mapping[index.Name]); } } }
public static void Import(BlittableJsonReaderObject indexDefinitionDoc, DocumentDatabase database, BuildVersionType buildType, bool removeAnalyzers) { var definition = ReadIndexDefinition(indexDefinitionDoc, buildType, out IndexType indexType); switch (indexType) { case IndexType.AutoMap: var autoMapIndexDefinition = (AutoMapIndexDefinition)definition; AsyncHelpers.RunSync(() => database.IndexStore.CreateIndex(autoMapIndexDefinition, RaftIdGenerator.NewId())); break; case IndexType.AutoMapReduce: var autoMapReduceIndexDefinition = (AutoMapReduceIndexDefinition)definition; AsyncHelpers.RunSync(() => database.IndexStore.CreateIndex(autoMapReduceIndexDefinition, RaftIdGenerator.NewId())); break; case IndexType.Map: case IndexType.MapReduce: case IndexType.JavaScriptMap: case IndexType.JavaScriptMapReduce: var indexDefinition = (IndexDefinition)definition; if (removeAnalyzers) { foreach (var indexDefinitionField in indexDefinition.Fields) { indexDefinitionField.Value.Analyzer = null; } } AsyncHelpers.RunSync(() => database.IndexStore.CreateIndex(indexDefinition, RaftIdGenerator.NewId())); break; default: throw new NotSupportedException(indexType.ToString()); } }
public void Cannot_Setup_Documents_Compression_When_Experimental_Mode_Is_Off() { DoNotReuseServer(new Dictionary <string, string> { { RavenConfiguration.GetKey(x => x.Core.FeaturesAvailability), FeaturesAvailability.Stable.ToString() } }); using (var store = GetDocumentStore()) { var databaseRecord = store.Maintenance.Server.Send(new GetDatabaseRecordOperation(store.Database)); Assert.Null(databaseRecord.DocumentsCompression); var databaseName = $"{store.Database}-{Guid.NewGuid()}"; using (EnsureDatabaseDeletion(databaseName, store)) { var e = Assert.Throws <RavenException>(() => store.Maintenance.Server.Send(new CreateDatabaseOperation(new DatabaseRecord { DatabaseName = databaseName, DocumentsCompression = new DocumentsCompressionConfiguration { CompressRevisions = true } }))); Assert.Contains("Can not use 'Documents Compression'", e.Message); store.Maintenance.Server.Send(new CreateDatabaseOperation(new DatabaseRecord { DatabaseName = databaseName })); using (var commands = store.Commands(databaseName)) { e = Assert.Throws <RavenException>(() => commands.ExecuteJson($"/admin/documents-compression/config?raft-request-id={RaftIdGenerator.NewId()}", HttpMethod.Post, new DocumentsCompressionConfiguration { CompressRevisions = true })); databaseRecord = store.Maintenance.Server.Send(new GetDatabaseRecordOperation(databaseName)); Assert.Null(databaseRecord.DocumentsCompression); } } } }
public void CanUseCustomSorter_Restart_Faulty() { var serverPath = NewDataPath(); var databasePath = NewDataPath(); IOExtensions.DeleteDirectory(serverPath); IOExtensions.DeleteDirectory(databasePath); var sorterName = GetDatabaseName(); using (var server = GetNewServer(new ServerCreationOptions { DataDirectory = serverPath, RunInMemory = false })) using (var store = GetDocumentStore(new Options { ModifyDatabaseName = _ => sorterName, Path = databasePath, RunInMemory = false, Server = server, DeleteDatabaseOnDispose = false })) { using (var session = store.OpenSession()) { session.Store(new Company { Name = "C1" }); session.Store(new Company { Name = "C2" }); session.SaveChanges(); } CanUseSorterInternal <SorterDoesNotExistException>(store, $"There is no sorter with '{sorterName}' name", $"There is no sorter with '{sorterName}' name", sorterName); var sorterCode = GetSorter("RavenDB_8355.MySorter.cs", "MySorter", sorterName); store.Maintenance.Server.Send(new PutServerWideSortersOperation(new SorterDefinition { Name = sorterName, Code = sorterCode })); CanUseSorterInternal <RavenException>(store, "Catch me: Name:2:0:False", "Catch me: Name:2:0:True", sorterName); server.ServerStore.DatabasesLandlord.UnloadDirectly(store.Database); // skipping compilation on purpose using (server.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext context)) using (var tx = context.OpenWriteTransaction()) { var command = new PutServerWideSorterCommand( new SorterDefinition { Name = sorterName, Code = sorterCode.Replace(sorterCode, "MySorter") }, RaftIdGenerator.NewId()); using (var json = context.ReadObject(command.ValueToJson(), command.Name)) { ClusterStateMachine.PutValueDirectly(context, command.Name, json, 1); } tx.Commit(); } } SorterCompilationCache.Instance.RemoveServerWideItem(sorterName); using (var server = GetNewServer(new ServerCreationOptions { DataDirectory = serverPath, RunInMemory = false, DeletePrevious = false })) using (var store = GetDocumentStore(new Options { ModifyDatabaseName = _ => sorterName, Path = databasePath, RunInMemory = false, Server = server, CreateDatabase = false })) { CanUseSorterInternal <RavenException>(store, "is an implementation of a faulty sorter", "is an implementation of a faulty sorter", sorterName); // can override with database sorter store.Maintenance.Send(new PutSortersOperation(new SorterDefinition { Name = sorterName, Code = GetSorter("RavenDB_8355.MySorter.cs", "MySorter", sorterName) })); CanUseSorterInternal <RavenException>(store, "Catch me: Name:2:0:False", "Catch me: Name:2:0:True", sorterName); // can go back to server analyzer store.Maintenance.Send(new DeleteSorterOperation(sorterName)); CanUseSorterInternal <RavenException>(store, "is an implementation of a faulty sorter", "is an implementation of a faulty sorter", sorterName); // can fix server sorter store.Maintenance.Server.Send(new PutServerWideSortersOperation(new SorterDefinition { Name = sorterName, Code = GetSorter("RavenDB_8355.MySorter.cs", "MySorter", sorterName) })); CanUseSorterInternal <RavenException>(store, "Catch me: Name:2:0:False", "Catch me: Name:2:0:True", sorterName); } }
public async Task SessionInSubscriptionsShouldNotTrackRevisions() { using (var store = GetDocumentStore()) { var subscriptionId = await store.Subscriptions.CreateAsync <Revision <User> >(); using (var context = JsonOperationContext.ShortTermSingleUse()) { var configuration = new RevisionsConfiguration { Default = new RevisionsCollectionConfiguration { Disabled = false, MinimumRevisionsToKeep = 5, }, Collections = new Dictionary <string, RevisionsCollectionConfiguration> { ["Users"] = new RevisionsCollectionConfiguration { Disabled = false } } }; await Server .ServerStore .ModifyDatabaseRevisions( context, store.Database, EntityToBlittable.ConvertCommandToBlittable(configuration, context), RaftIdGenerator.NewId()); } for (int i = 0; i < 10; i++) { for (var j = 0; j < 10; j++) { using (var session = store.OpenSession()) { session.Store(new User { Name = $"users{i} ver {j}" }, "users/" + i); session.SaveChanges(); } } } using (var sub = store.Subscriptions.GetSubscriptionWorker <Revision <User> >(new SubscriptionWorkerOptions(subscriptionId) { TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5) })) { Exception exception = null; var mre = new AsyncManualResetEvent(); GC.KeepAlive(sub.Run(x => { try { using (var session = x.OpenSession()) { x.Items[0].Result.Current.Name = "aaaa"; session.SaveChanges(); } } catch (Exception e) { exception = e; } finally { mre.Set(); } })); Assert.True(await mre.WaitAsync(_reasonableWaitTime)); if (exception != null) { throw exception; } } } }
private static async Task GetIdentitiesValues(JsonOperationContext ctx, DocumentDatabase database, ServerStore serverStore, List <string> identities, List <int> positionInListToCommandIndex, CommandData[] cmds) { var newIds = await serverStore.GenerateClusterIdentitiesBatchAsync(database.Name, identities, RaftIdGenerator.NewId()); Debug.Assert(newIds.Count == identities.Count); var emptyChangeVector = ctx.GetLazyString(""); for (var index = 0; index < positionInListToCommandIndex.Count; index++) { var value = positionInListToCommandIndex[index]; cmds[value].Id = cmds[value].Id.Substring(0, cmds[value].Id.Length - 1) + "/" + newIds[index]; if (string.IsNullOrEmpty(cmds[value].ChangeVector) == false) { ThrowInvalidUsageOfChangeVectorWithIdentities(cmds[value]); } cmds[value].ChangeVector = emptyChangeVector; } }
public async Task CanReplaceClusterCertWithExtensionPoint() { string loadAndRenewScript; string certChangedScript; IDictionary <string, string> customSettings1 = new ConcurrentDictionary <string, string>(); IDictionary <string, string> customSettings2 = new ConcurrentDictionary <string, string>(); IDictionary <string, string> customSettings3 = new ConcurrentDictionary <string, string>(); var certPath = GenerateAndSaveSelfSignedCertificate(); var cert2Path = GenerateAndSaveSelfSignedCertificate(createNew: true); var outputFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".txt")); var firstServerCert = new X509Certificate2(certPath, (string)null, X509KeyStorageFlags.Exportable | X509KeyStorageFlags.MachineKeySet); var newServerCert = new X509Certificate2(cert2Path, (string)null, X509KeyStorageFlags.Exportable | X509KeyStorageFlags.MachineKeySet); if (PlatformDetails.RunningOnPosix) { var loadAndRenewScriptNode1Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".sh")); var loadAndRenewScriptNode2Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".sh")); var loadAndRenewScriptNode3Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".sh")); var certChangedScriptNode1Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".sh")); var certChangedScriptNode2Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".sh")); var certChangedScriptNode3Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".sh")); var loadArgsNode1 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> { loadAndRenewScriptNode1Path, certPath }); var renewArgsNode1 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> { loadAndRenewScriptNode1Path, cert2Path }); var certChangedArgsNode1 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> { certChangedScriptNode1Path, outputFile }); var loadArgsNode2 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> { loadAndRenewScriptNode2Path, certPath }); var renewArgsNode2 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> { loadAndRenewScriptNode2Path, cert2Path }); var certChangedArgsNode2 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> { certChangedScriptNode2Path, outputFile }); var loadArgsNode3 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> { loadAndRenewScriptNode3Path, certPath }); var renewArgsNode3 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> { loadAndRenewScriptNode3Path, cert2Path }); var certChangedArgsNode3 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> { certChangedScriptNode3Path, outputFile }); customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExec)] = "bash"; customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExecArguments)] = $"{loadArgsNode1}"; customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExec)] = "bash"; customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExecArguments)] = $"{renewArgsNode1}"; customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExec)] = "bash"; customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExecArguments)] = $"{certChangedArgsNode1}"; customSettings1[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = "https://" + Environment.MachineName + ":0"; customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExec)] = "bash"; customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExecArguments)] = $"{loadArgsNode2}"; customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExec)] = "bash"; customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExecArguments)] = $"{renewArgsNode2}"; customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExec)] = "bash"; customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExecArguments)] = $"{certChangedArgsNode2}"; customSettings2[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = "https://" + Environment.MachineName + ":0"; customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExec)] = "bash"; customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExecArguments)] = $"{loadArgsNode3}"; customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExec)] = "bash"; customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExecArguments)] = $"{renewArgsNode3}"; customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExec)] = "bash"; customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExecArguments)] = $"{certChangedArgsNode3}"; customSettings3[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = "https://" + Environment.MachineName + ":0"; loadAndRenewScript = "#!/bin/bash\ncat -u \"$1\""; certChangedScript = "#!/bin/bash\nwhile read line; do\n\techo \"$line\" >> \"$1\"\ndone < /dev/stdin"; File.WriteAllText(loadAndRenewScriptNode1Path, loadAndRenewScript); File.WriteAllText(loadAndRenewScriptNode2Path, loadAndRenewScript); File.WriteAllText(loadAndRenewScriptNode3Path, loadAndRenewScript); File.WriteAllText(certChangedScriptNode1Path, certChangedScript); File.WriteAllText(certChangedScriptNode2Path, certChangedScript); File.WriteAllText(certChangedScriptNode3Path, certChangedScript); Process.Start("chmod", $"700 {loadAndRenewScriptNode1Path}"); Process.Start("chmod", $"700 {loadAndRenewScriptNode2Path}"); Process.Start("chmod", $"700 {loadAndRenewScriptNode3Path}"); Process.Start("chmod", $"700 {certChangedScriptNode1Path}"); Process.Start("chmod", $"700 {certChangedScriptNode2Path}"); Process.Start("chmod", $"700 {certChangedScriptNode3Path}"); } else { var loadAndRenewScriptNode1Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".ps1")); var loadAndRenewScriptNode2Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".ps1")); var loadAndRenewScriptNode3Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".ps1")); var certChangedScriptNode1Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".ps1")); var certChangedScriptNode2Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".ps1")); var certChangedScriptNode3Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".ps1")); var loadArgsNode1 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> { "-NoProfile", loadAndRenewScriptNode1Path, certPath }); var renewArgsNode1 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> { "-NoProfile", loadAndRenewScriptNode1Path, cert2Path }); var certChangedArgsNode1 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> { "-NoProfile", certChangedScriptNode1Path, outputFile }); var loadArgsNode2 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> { "-NoProfile", loadAndRenewScriptNode2Path, certPath }); var renewArgsNode2 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> { "-NoProfile", loadAndRenewScriptNode2Path, cert2Path }); var certChangedArgsNode2 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> { "-NoProfile", certChangedScriptNode2Path, outputFile }); var loadArgsNode3 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> { "-NoProfile", loadAndRenewScriptNode3Path, certPath }); var renewArgsNode3 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> { "-NoProfile", loadAndRenewScriptNode3Path, cert2Path }); var certChangedArgsNode3 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> { "-NoProfile", certChangedScriptNode3Path, outputFile }); customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExec)] = "powershell"; customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExecArguments)] = $"{loadArgsNode1}"; customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExec)] = "powershell"; customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExecArguments)] = $"{renewArgsNode1}"; customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExec)] = "powershell"; customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExecArguments)] = $"{certChangedArgsNode1}"; customSettings1[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = "https://" + Environment.MachineName + ":0"; customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExec)] = "powershell"; customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExecArguments)] = $"{loadArgsNode2}"; customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExec)] = "powershell"; customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExecArguments)] = $"{renewArgsNode2}"; customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExec)] = "powershell"; customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExecArguments)] = $"{certChangedArgsNode2}"; customSettings2[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = "https://" + Environment.MachineName + ":0"; customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExec)] = "powershell"; customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExecArguments)] = $"{loadArgsNode3}"; customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExec)] = "powershell"; customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExecArguments)] = $"{renewArgsNode3}"; customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExec)] = "powershell"; customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExecArguments)] = $"{certChangedArgsNode3}"; customSettings3[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = "https://" + Environment.MachineName + ":0"; loadAndRenewScript = @"param([string]$userArg1) try { $bytes = Get-Content -path $userArg1 -encoding Byte $stdout = [System.Console]::OpenStandardOutput() $stdout.Write($bytes, 0, $bytes.Length) } catch { Write-Error $_.Exception exit 1 } exit 0"; certChangedScript = @"param([string]$userArg1) try { $certBase64 = [System.Console]::ReadLine() Add-Content $userArg1 $certBase64 } catch { Write-Error $_.Exception exit 1 } exit 0"; File.WriteAllText(loadAndRenewScriptNode1Path, loadAndRenewScript); File.WriteAllText(loadAndRenewScriptNode2Path, loadAndRenewScript); File.WriteAllText(loadAndRenewScriptNode3Path, loadAndRenewScript); File.WriteAllText(certChangedScriptNode1Path, certChangedScript); File.WriteAllText(certChangedScriptNode2Path, certChangedScript); File.WriteAllText(certChangedScriptNode3Path, certChangedScript); } var clusterSize = 3; var databaseName = GetDatabaseName(); var leader = await CreateRaftClusterAndGetLeader(clusterSize, false, useSsl : true, customSettingsList : new List <IDictionary <string, string> > { customSettings1, customSettings2, customSettings3 }); Assert.True(leader?.Certificate?.Certificate?.Thumbprint?.Equals(firstServerCert.Thumbprint), "Cert is identical"); var adminCertificate = AskServerForClientCertificate(certPath, new Dictionary <string, DatabaseAccess>(), SecurityClearance.ClusterAdmin, server: leader); DatabasePutResult databaseResult; using (var store = new DocumentStore { Urls = new[] { leader.WebUrl }, Database = databaseName, Certificate = adminCertificate, Conventions = { DisableTopologyUpdates = true } }.Initialize()) { var doc = new DatabaseRecord(databaseName); databaseResult = await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(doc, clusterSize)); } Assert.Equal(clusterSize, databaseResult.Topology.AllNodes.Count()); foreach (var server in Servers) { await server.ServerStore.Cluster.WaitForIndexNotification(databaseResult.RaftCommandIndex); } foreach (var server in Servers.Where(s => databaseResult.NodesAddedTo.Any(n => n == s.WebUrl))) { await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName); } using (var store = new DocumentStore() { Urls = new[] { databaseResult.NodesAddedTo[0] }, Database = databaseName, Certificate = adminCertificate, Conventions = { DisableTopologyUpdates = true } }.Initialize()) { using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "Karmelush" }, "users/1"); await session.SaveChangesAsync(); } var mre = new ManualResetEventSlim(); leader.ServerCertificateChanged += (sender, args) => mre.Set(); // This will initiate the refresh cycle, and ask a new certificate from the executable leader.RefreshClusterCertificate(false, RaftIdGenerator.NewId()); Assert.True(mre.Wait(Debugger.IsAttached ? TimeSpan.FromMinutes(10) : TimeSpan.FromMinutes(2)), "Waited too long"); Assert.NotNull(leader.Certificate.Certificate.Thumbprint); Assert.True(leader.Certificate.Certificate.Thumbprint.Equals(newServerCert.Thumbprint), "New cert is identical"); var base64NewCertWrittenByExecutable = File.ReadAllLines(outputFile)[0]; var loadedCertificate = new X509Certificate2(Convert.FromBase64String(base64NewCertWrittenByExecutable)); Assert.Equal(newServerCert.Thumbprint, loadedCertificate.Thumbprint); using (var session = store.OpenSession()) { var user1 = session.Load <User>("users/1"); Assert.NotNull(user1); Assert.Equal("Karmelush", user1.Name); } } }
private async Task AddDatabases() { await _locker.WaitAsync(); try { using (_server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) { context.OpenReadTransaction(); var databases = _server .ServerStore .Cluster .ItemKeysStartingWith(context, Constants.Documents.Prefix, 0, int.MaxValue) .Select(x => x.Substring(Constants.Documents.Prefix.Length)) .ToList(); if (databases.Count == 0) { return; } var mapping = GetMapping(_server.ServerStore, context); var missingDatabases = new List <string>(); foreach (var database in databases) { if (mapping.ContainsKey(database) == false) { missingDatabases.Add(database); } } if (missingDatabases.Count > 0) { context.CloseTransaction(); var result = await _server.ServerStore.SendToLeaderAsync(new UpdateSnmpDatabasesMappingCommand(missingDatabases, RaftIdGenerator.NewId())); await _server.ServerStore.Cluster.WaitForIndexNotification(result.Index); context.OpenReadTransaction(); mapping = GetMapping(_server.ServerStore, context); } foreach (var database in databases) { LoadDatabase(database, mapping[database]); } } } finally { _locker.Release(); } }
public async Task <IOperationResult> Execute(Action <IOperationProgress> onProgress) { var databaseName = RestoreFromConfiguration.DatabaseName; var result = new RestoreResult { DataDirectory = RestoreFromConfiguration.DataDirectory }; try { var filesToRestore = await GetOrderedFilesToRestore(); using (_serverStore.ContextPool.AllocateOperationContext(out JsonOperationContext serverContext)) { if (onProgress == null) { onProgress = _ => { } } ; Stopwatch sw = null; RestoreSettings restoreSettings = null; var firstFile = filesToRestore[0]; var extension = Path.GetExtension(firstFile); var snapshotRestore = false; if ((extension == Constants.Documents.PeriodicBackup.SnapshotExtension) || (extension == Constants.Documents.PeriodicBackup.EncryptedSnapshotExtension)) { onProgress.Invoke(result.Progress); snapshotRestore = true; sw = Stopwatch.StartNew(); if (extension == Constants.Documents.PeriodicBackup.EncryptedSnapshotExtension) { _hasEncryptionKey = RestoreFromConfiguration.EncryptionKey != null || RestoreFromConfiguration.BackupEncryptionSettings?.Key != null; } // restore the snapshot restoreSettings = await SnapshotRestore(serverContext, firstFile, onProgress, result); if (restoreSettings != null && RestoreFromConfiguration.SkipIndexes) { // remove all indexes from the database record restoreSettings.DatabaseRecord.AutoIndexes = null; restoreSettings.DatabaseRecord.Indexes = null; } // removing the snapshot from the list of files filesToRestore.RemoveAt(0); } else { result.SnapshotRestore.Skipped = true; result.SnapshotRestore.Processed = true; onProgress.Invoke(result.Progress); } if (restoreSettings == null) { restoreSettings = new RestoreSettings { DatabaseRecord = new DatabaseRecord(databaseName) { // we only have a smuggler restore // use the encryption key to encrypt the database Encrypted = _hasEncryptionKey } }; DatabaseHelper.Validate(databaseName, restoreSettings.DatabaseRecord, _serverStore.Configuration); } var databaseRecord = restoreSettings.DatabaseRecord; if (databaseRecord.Settings == null) { databaseRecord.Settings = new Dictionary <string, string>(); } var runInMemoryConfigurationKey = RavenConfiguration.GetKey(x => x.Core.RunInMemory); databaseRecord.Settings.Remove(runInMemoryConfigurationKey); if (_serverStore.Configuration.Core.RunInMemory) { databaseRecord.Settings[runInMemoryConfigurationKey] = "false"; } var dataDirectoryConfigurationKey = RavenConfiguration.GetKey(x => x.Core.DataDirectory); databaseRecord.Settings.Remove(dataDirectoryConfigurationKey); // removing because we want to restore to given location, not to serialized in backup one if (_restoringToDefaultDataDirectory == false) { databaseRecord.Settings[dataDirectoryConfigurationKey] = RestoreFromConfiguration.DataDirectory; } if (_hasEncryptionKey) { // save the encryption key so we'll be able to access the database _serverStore.PutSecretKey(RestoreFromConfiguration.EncryptionKey, databaseName, overwrite: false); } var addToInitLog = new Action <string>(txt => // init log is not save in mem during RestoreBackup { var msg = $"[RestoreBackup] {DateTime.UtcNow} :: Database '{databaseName}' : {txt}"; if (Logger.IsInfoEnabled) { Logger.Info(msg); } }); var configuration = _serverStore .DatabasesLandlord .CreateDatabaseConfiguration(databaseName, ignoreDisabledDatabase: true, ignoreBeenDeleted: true, ignoreNotRelevant: true, databaseRecord); using (var database = new DocumentDatabase(databaseName, configuration, _serverStore, addToInitLog)) { // smuggler needs an existing document database to operate var options = InitializeOptions.SkipLoadingDatabaseRecord; if (snapshotRestore) { options |= InitializeOptions.GenerateNewDatabaseId; } database.Initialize(options); databaseRecord.Topology = new DatabaseTopology(); // restoring to the current node only databaseRecord.Topology.Members.Add(_nodeTag); // we are currently restoring, shouldn't try to access it databaseRecord.DatabaseState = DatabaseStateStatus.RestoreInProgress; var(index, _) = await _serverStore.WriteDatabaseRecordAsync(databaseName, databaseRecord, null, RaftIdGenerator.NewId(), restoreSettings.DatabaseValues, isRestore : true); await _serverStore.Cluster.WaitForIndexNotification(index); DisableOngoingTasksIfNeeded(databaseRecord); using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { if (snapshotRestore) { await RestoreFromSmugglerFile(onProgress, database, firstFile, context); await SmugglerRestore(database, filesToRestore, context, databaseRecord, onProgress, result); result.SnapshotRestore.Processed = true; var summary = database.GetDatabaseSummary(); result.Documents.ReadCount += summary.DocumentsCount; result.Documents.Attachments.ReadCount += summary.AttachmentsCount; result.Counters.ReadCount += summary.CounterEntriesCount; result.RevisionDocuments.ReadCount += summary.RevisionsCount; result.Conflicts.ReadCount += summary.ConflictsCount; result.Indexes.ReadCount += databaseRecord.GetIndexesCount(); result.CompareExchange.ReadCount += summary.CompareExchangeCount; result.CompareExchangeTombstones.ReadCount += summary.CompareExchangeTombstonesCount; result.Identities.ReadCount += summary.IdentitiesCount; result.AddInfo($"Successfully restored {result.SnapshotRestore.ReadCount} files during snapshot restore, took: {sw.ElapsedMilliseconds:#,#;;0}ms"); onProgress.Invoke(result.Progress); } else { await SmugglerRestore(database, filesToRestore, context, databaseRecord, onProgress, result); } result.DatabaseRecord.Processed = true; result.Documents.Processed = true; result.RevisionDocuments.Processed = true; result.Conflicts.Processed = true; result.Indexes.Processed = true; result.Counters.Processed = true; result.Identities.Processed = true; result.CompareExchange.Processed = true; result.Subscriptions.Processed = true; onProgress.Invoke(result.Progress); } } // after the db for restore is done, we can safely set the db state to normal and write the DatabaseRecord databaseRecord.DatabaseState = DatabaseStateStatus.Normal; var(updateIndex, _) = await _serverStore.WriteDatabaseRecordAsync(databaseName, databaseRecord, null, RaftIdGenerator.DontCareId, isRestore : true); await _serverStore.Cluster.WaitForIndexNotification(updateIndex); if (databaseRecord.Topology.RelevantFor(_serverStore.NodeTag)) { // we need to wait for the database record change to be propagated properly var db = await _serverStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName); await db.RachisLogIndexNotifications.WaitForIndexNotification(updateIndex, _operationCancelToken.Token); } return(result); } } catch (Exception e) { if (Logger.IsOperationsEnabled) { Logger.Operations("Failed to restore database", e); } var alert = AlertRaised.Create( RestoreFromConfiguration.DatabaseName, "Failed to restore database", $"Could not restore database named {RestoreFromConfiguration.DatabaseName}", AlertType.RestoreError, NotificationSeverity.Error, details: new ExceptionDetails(e)); _serverStore.NotificationCenter.Add(alert); using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) { bool databaseExists; using (context.OpenReadTransaction()) { databaseExists = _serverStore.Cluster.DatabaseExists(context, RestoreFromConfiguration.DatabaseName); } if (databaseExists == false) { // delete any files that we already created during the restore IOExtensions.DeleteDirectory(RestoreFromConfiguration.DataDirectory); } else { var deleteResult = await _serverStore.DeleteDatabaseAsync(RestoreFromConfiguration.DatabaseName, true, new[] { _serverStore.NodeTag }, RaftIdGenerator.DontCareId); await _serverStore.Cluster.WaitForIndexNotification(deleteResult.Index); } } result.AddError($"Error occurred during restore of database {databaseName}. Exception: {e.Message}"); onProgress.Invoke(result.Progress); throw; } finally { Dispose(); } }