private void WriteStatus(PeriodicBackupStatus status)
{
AddInfo("Saving backup status");
try
{
var command = new UpdatePeriodicBackupStatusCommand(_database.Name, RaftIdGenerator.NewId())
{
PeriodicBackupStatus = status
};
var result = AsyncHelpers.RunSync(() => _serverStore.SendToLeaderAsync(command));
if (_logger.IsInfoEnabled)
{
_logger.Info($"Periodic backup status with task id {status.TaskId} was updated");
}
AsyncHelpers.RunSync(() => _serverStore.WaitForCommitIndexChange(RachisConsensus.CommitIndexModification.GreaterOrEqual, result.Index));
}
catch (Exception e)
{
const string message = "Error saving the periodic backup status";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
}
}
public async Task Command_sent_twice_should_not_throw_timeout_error()
{
var leader = await CreateNetworkAndGetLeader(3);
var nonLeader = GetRandomFollower();
long lastIndex;
using (leader.ContextPool.AllocateOperationContext(out ClusterOperationContext context))
{
var cmd = new TestCommandWithRaftId("test", RaftIdGenerator.NewId())
{
RaftCommandIndex = 322
};
var t = leader.PutAsync(cmd);
await leader.PutAsync(cmd);
// this should not throw timeout exception.
var exception = await Record.ExceptionAsync(async() => await t);
Assert.Null(exception);
using (context.OpenReadTransaction())
lastIndex = leader.GetLastEntryIndex(context);
}
var waitForAllCommits = nonLeader.WaitForCommitIndexChange(RachisConsensus.CommitIndexModification.GreaterOrEqual, lastIndex);
Assert.True(await waitForAllCommits.WaitAsync(LongWaitTime), "didn't commit in time");
}
public async Task RemoveEntryFromRaftLogEP()
{
var leader = await CreateRaftClusterAndGetLeader(3);
var database = GetDatabaseName();
await CreateDatabaseInClusterInner(new DatabaseRecord(database), 3, leader.WebUrl, null);
using (var store = new DocumentStore {
Database = database, Urls = new[] { leader.WebUrl }
}.Initialize())
{
leader.ServerStore.Engine.StateMachine.Validator = new TestCommandValidator();
var documentDatabase = await leader.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database);
var cmd = new RachisConsensusTestBase.TestCommandWithRaftId("test", RaftIdGenerator.NewId());
_ = leader.ServerStore.Engine.CurrentLeader.PutAsync(cmd, new TimeSpan(2000));
var cmd2 = new CreateDatabaseOperation.CreateDatabaseCommand(new DatabaseRecord("Toli"), 1);
_ = leader.ServerStore.SendToLeaderAsync(new AddDatabaseCommand(Guid.NewGuid().ToString())
{
Record = new DatabaseRecord("Toli")
{
Topology = new DatabaseTopology
{
Members = new List <string> {
"A", "B", "C"
},
Rehabs = new List <string> {
},
ReplicationFactor = 3
}
},
Name = "Toli"
});
foreach (var server in Servers)
{
Assert.False(leader.ServerStore.DatabasesLandlord.IsDatabaseLoaded("Toli"));
}
long index;
using (documentDatabase.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext context))
using (var tx = context.OpenReadTransaction())
{
leader.ServerStore.Engine.GetLastCommitIndex(context, out index, out long term);
}
var operation = await store.Maintenance.SendAsync(new RemoveEntryFromRaftLogOperation(index + 1));
foreach (var server in Servers)
{
Assert.Contains(server.ServerStore.NodeTag, operation);
var val = WaitForValueAsync(() => server.ServerStore.DatabasesLandlord.IsDatabaseLoaded("Toli"), true);
Assert.True(val.Result);
}
}
}
public async Task <(Index Index, bool HasCreatedAutoIndex)> CreateAutoIndexIfNeeded(IndexQueryServerSide query, bool createAutoIndexIfNoMatchIsFound, TimeSpan?customStalenessWaitTimeout, CancellationToken token)
{
var map = DynamicQueryMapping.Create(query);
bool hasCreatedAutoIndex = false;
Index index;
while (TryMatchExistingIndexToQuery(map, out index) == false)
{
if (createAutoIndexIfNoMatchIsFound == false)
{
throw new IndexDoesNotExistException("Could not find index for a given query.");
}
var definition = map.CreateAutoIndexDefinition();
index = await _indexStore.CreateIndex(definition, RaftIdGenerator.NewId());
if (index == null)
{
// the index was deleted, we'll try to find a better match (replaced by a wider index)
continue;
}
hasCreatedAutoIndex = true;
if (query.WaitForNonStaleResultsTimeout.HasValue == false)
{
query.WaitForNonStaleResultsTimeout = customStalenessWaitTimeout ?? TimeSpan.FromSeconds(15);
}
var t = CleanupSupersededAutoIndexes(index, map, RaftIdGenerator.NewId(), token)
.ContinueWith(task =>
{
if (task.Exception != null)
{
if (token.IsCancellationRequested)
{
return;
}
if (_indexStore.Logger.IsInfoEnabled)
{
_indexStore.Logger.Info("Failed to delete superseded indexes for index " + index.Name);
}
}
}, token);
if (query.WaitForNonStaleResults &&
Database.Configuration.Indexing.TimeToWaitBeforeDeletingAutoIndexMarkedAsIdle.AsTimeSpan ==
TimeSpan.Zero)
{
await t; // this is used in testing, mainly
}
break;
}
return(index, hasCreatedAutoIndex);
}
private void AddDatabaseIfNecessary(string databaseName)
{
if (string.IsNullOrWhiteSpace(databaseName))
{
return;
}
if (_loadedDatabases.ContainsKey(databaseName))
{
return;
}
Task.Factory.StartNew(async() =>
{
await _locker.WaitAsync();
try
{
if (_loadedDatabases.ContainsKey(databaseName))
{
return;
}
using (_server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
context.OpenReadTransaction();
var mapping = GetMapping(_server.ServerStore, context);
if (mapping.ContainsKey(databaseName) == false)
{
context.CloseTransaction();
var result = await _server.ServerStore.SendToLeaderAsync(new UpdateSnmpDatabasesMappingCommand(new List <string> {
databaseName
}, RaftIdGenerator.NewId()));
await _server.ServerStore.Cluster.WaitForIndexNotification(result.Index);
context.OpenReadTransaction();
mapping = GetMapping(_server.ServerStore, context);
}
LoadDatabase(databaseName, mapping[databaseName]);
}
}
catch (Exception e)
{
if (Logger.IsOperationsEnabled)
{
Logger.Operations($"Failed to update the SNMP mapping for database: {databaseName}", e);
}
}
finally
{
_locker.Release();
}
});
}
private async Task SaveDatabaseRecordAsync(string databaseName, DatabaseRecord databaseRecord, Dictionary <string,
BlittableJsonReaderObject> databaseValues, RestoreResult restoreResult, Action <IOperationProgress> onProgress)
{
// at this point we restored a large portion of the database or all of it
// we'll retry saving the database record since a failure here will cause us to abort the entire restore operation
var index = await RunWithRetries(async() =>
{
var result = await _serverStore.WriteDatabaseRecordAsync(
databaseName, databaseRecord, null, RaftIdGenerator.NewId(), databaseValues, isRestore: true);
return(result.Index);
},
"Saving the database record",
"Failed to save the database record, the restore is aborted");
await RunWithRetries(async() =>
{
await _serverStore.Cluster.WaitForIndexNotification(index, TimeSpan.FromSeconds(30));
return(index);
},
$"Verifying that the change to the database record propagated to node {_serverStore.NodeTag}",
$"Failed to verify that the change to the database record was propagated to node {_serverStore.NodeTag}, the restore is aborted");
async Task <long> RunWithRetries(Func <Task <long> > action, string infoMessage, string errorMessage)
{
const int maxRetries = 10;
var retries = 0;
while (true)
{
try
{
_operationCancelToken.Token.ThrowIfCancellationRequested();
restoreResult.AddInfo(infoMessage);
onProgress.Invoke(restoreResult.Progress);
return(await action());
}
catch (TimeoutException)
{
if (++retries < maxRetries)
{
continue;
}
restoreResult.AddError(errorMessage);
onProgress.Invoke(restoreResult.Progress);
throw;
}
}
}
}
public async Task UpdateClientConnectionTime(long id, string name, string mentorNode = null)
{
var command = new UpdateSubscriptionClientConnectionTime(_db.Name, RaftIdGenerator.NewId())
{
NodeTag = _serverStore.NodeTag,
HasHighlyAvailableTasks = _serverStore.LicenseManager.HasHighlyAvailableTasks(),
SubscriptionName = name,
LastClientConnectionTime = DateTime.UtcNow
};
var(etag, _) = await _serverStore.SendToLeaderAsync(command);
await _db.RachisLogIndexNotifications.WaitForIndexNotification(etag, _serverStore.Engine.OperationTimeout);
}
private void AddIndexIfNecessary(IndexChange change)
{
if (change.Type != IndexChangeTypes.IndexAdded)
{
return;
}
Task.Factory.StartNew(async() =>
{
await _locker.WaitAsync();
try
{
if (_loadedIndexes.ContainsKey(change.Name))
{
return;
}
var database = await _databaseLandlord.TryGetOrCreateResourceStore(_databaseName);
using (database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
context.OpenReadTransaction();
var mapping = GetIndexMapping(context, database.ServerStore, database.Name);
if (mapping.ContainsKey(change.Name) == false)
{
context.CloseTransaction();
var result = await database.ServerStore.SendToLeaderAsync(new UpdateSnmpDatabaseIndexesMappingCommand(_databaseName, new List <string>
{
change.Name
}, RaftIdGenerator.NewId()));
await database.ServerStore.Cluster.WaitForIndexNotification(result.Index);
context.OpenReadTransaction();
mapping = GetIndexMapping(context, database.ServerStore, database.Name);
}
LoadIndex(change.Name, (int)mapping[change.Name]);
}
}
finally
{
_locker.Release();
}
});
}
public async Task AcknowledgeBatchProcessed(long id, string name, string changeVector, string previousChangeVector)
{
var command = new AcknowledgeSubscriptionBatchCommand(_db.Name, RaftIdGenerator.NewId())
{
ChangeVector = changeVector,
NodeTag = _serverStore.NodeTag,
HasHighlyAvailableTasks = _serverStore.LicenseManager.HasHighlyAvailableTasks(),
SubscriptionId = id,
SubscriptionName = name,
LastTimeServerMadeProgressWithDocuments = DateTime.UtcNow,
LastKnownSubscriptionChangeVector = previousChangeVector
};
var(etag, _) = await _serverStore.SendToLeaderAsync(command);
await _db.RachisLogIndexNotifications.WaitForIndexNotification(etag, _serverStore.Engine.OperationTimeout);
}
protected string GetRaftRequestIdFromQuery()
{
var guid = GetStringQueryString("raft-request-id", required: false);
if (guid == null)
{
#if DEBUG
var fromStudio = HttpContext.Request.Headers.ContainsKey(Constants.Headers.StudioVersion);
if (fromStudio)
{
guid = RaftIdGenerator.NewId();
}
#else
guid = RaftIdGenerator.NewId();
#endif
}
return(guid);
}
public async Task SearchForPageShouldNotSkipLastPage()
{
using (var inputStream = GetDump("RavenDB-13937.ravendbdump"))
using (var stream = new GZipStream(inputStream, CompressionMode.Decompress))
{
using (var database = CreateDocumentDatabase())
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var source = new StreamSource(stream, context, database))
{
var editRevisions = new EditRevisionsConfigurationCommand(new RevisionsConfiguration
{
Collections = new Dictionary <string, RevisionsCollectionConfiguration>
{
["Orders"] = new RevisionsCollectionConfiguration
{
Disabled = false
}
}
}, database.Name, RaftIdGenerator.NewId());
var(index, _) = await database.ServerStore.SendToLeaderAsync(editRevisions);
await database.RachisLogIndexNotifications.WaitForIndexNotification(index, database.ServerStore.Engine.OperationTimeout);
var destination = new DatabaseDestination(database);
var smuggler = await(new DatabaseSmuggler(database, source, destination, database.Time, new DatabaseSmugglerOptionsServerSide
{
OperateOnTypes = DatabaseItemType.Documents | DatabaseItemType.RevisionDocuments | DatabaseItemType.Attachments |
DatabaseItemType.Indexes,
SkipRevisionCreation = true
}).ExecuteAsync());
using (context.OpenReadTransaction())
{
var(revisions, count) = database.DocumentsStorage.RevisionsStorage.GetRevisions(context, "Orders/825-A", 0, int.MaxValue);
Assert.Equal(count, revisions.Length);
}
}
}
}
private void NotifyLeaderAboutRemoval(string dbName, string databaseId)
{
var cmd = new RemoveNodeFromDatabaseCommand(dbName, databaseId, RaftIdGenerator.NewId())
{
NodeTag = _serverStore.NodeTag
};
_serverStore.SendToLeaderAsync(cmd)
.ContinueWith(async t =>
{
var message = $"Failed to notify leader about removal of node {_serverStore.NodeTag} from database '{dbName}', will retry again in 15 seconds.";
if (t.IsFaulted)
{
var ex = t.Exception.ExtractSingleInnerException();
if (ex is DatabaseDoesNotExistException)
{
return;
}
if (_logger.IsInfoEnabled)
{
_logger.Info(message, t.Exception);
}
}
if (t.IsCanceled)
{
if (_logger.IsInfoEnabled)
{
_logger.Info(message, t.Exception);
}
}
await Task.Delay(TimeSpan.FromSeconds(15));
NotifyLeaderAboutRemoval(dbName, databaseId);
}, TaskContinuationOptions.NotOnRanToCompletion);
}
private async Task AddIndexesFromDatabase(DocumentDatabase database)
{
var indexes = database.IndexStore.GetIndexes().ToList();
if (indexes.Count == 0)
{
return;
}
using (database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
context.OpenReadTransaction();
var mapping = GetIndexMapping(context, database.ServerStore, database.Name);
var missingIndexes = new List <string>();
foreach (var index in indexes)
{
if (mapping.ContainsKey(index.Name) == false)
{
missingIndexes.Add(index.Name);
}
}
if (missingIndexes.Count > 0)
{
context.CloseTransaction();
var result = await database.ServerStore.SendToLeaderAsync(new UpdateSnmpDatabaseIndexesMappingCommand(database.Name, missingIndexes, RaftIdGenerator.NewId()));
await database.ServerStore.Cluster.WaitForIndexNotification(result.Index);
context.OpenReadTransaction();
mapping = GetIndexMapping(context, database.ServerStore, database.Name);
}
foreach (var index in indexes)
{
LoadIndex(index.Name, (int)mapping[index.Name]);
}
}
}
public static void Import(BlittableJsonReaderObject indexDefinitionDoc, DocumentDatabase database, BuildVersionType buildType, bool removeAnalyzers)
{
var definition = ReadIndexDefinition(indexDefinitionDoc, buildType, out IndexType indexType);
switch (indexType)
{
case IndexType.AutoMap:
var autoMapIndexDefinition = (AutoMapIndexDefinition)definition;
AsyncHelpers.RunSync(() => database.IndexStore.CreateIndex(autoMapIndexDefinition, RaftIdGenerator.NewId()));
break;
case IndexType.AutoMapReduce:
var autoMapReduceIndexDefinition = (AutoMapReduceIndexDefinition)definition;
AsyncHelpers.RunSync(() => database.IndexStore.CreateIndex(autoMapReduceIndexDefinition, RaftIdGenerator.NewId()));
break;
case IndexType.Map:
case IndexType.MapReduce:
case IndexType.JavaScriptMap:
case IndexType.JavaScriptMapReduce:
var indexDefinition = (IndexDefinition)definition;
if (removeAnalyzers)
{
foreach (var indexDefinitionField in indexDefinition.Fields)
{
indexDefinitionField.Value.Analyzer = null;
}
}
AsyncHelpers.RunSync(() => database.IndexStore.CreateIndex(indexDefinition, RaftIdGenerator.NewId()));
break;
default:
throw new NotSupportedException(indexType.ToString());
}
}
public void CanUseCustomSorter_Restart_Faulty()
{
var serverPath = NewDataPath();
var databasePath = NewDataPath();
IOExtensions.DeleteDirectory(serverPath);
IOExtensions.DeleteDirectory(databasePath);
var sorterName = GetDatabaseName();
using (var server = GetNewServer(new ServerCreationOptions
{
DataDirectory = serverPath,
RunInMemory = false
}))
using (var store = GetDocumentStore(new Options
{
ModifyDatabaseName = _ => sorterName,
Path = databasePath,
RunInMemory = false,
Server = server,
DeleteDatabaseOnDispose = false
}))
{
using (var session = store.OpenSession())
{
session.Store(new Company {
Name = "C1"
});
session.Store(new Company {
Name = "C2"
});
session.SaveChanges();
}
CanUseSorterInternal <SorterDoesNotExistException>(store, $"There is no sorter with '{sorterName}' name", $"There is no sorter with '{sorterName}' name", sorterName);
var sorterCode = GetSorter("RavenDB_8355.MySorter.cs", "MySorter", sorterName);
store.Maintenance.Server.Send(new PutServerWideSortersOperation(new SorterDefinition
{
Name = sorterName,
Code = sorterCode
}));
CanUseSorterInternal <RavenException>(store, "Catch me: Name:2:0:False", "Catch me: Name:2:0:True", sorterName);
server.ServerStore.DatabasesLandlord.UnloadDirectly(store.Database);
// skipping compilation on purpose
using (server.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext context))
using (var tx = context.OpenWriteTransaction())
{
var command = new PutServerWideSorterCommand(
new SorterDefinition {
Name = sorterName, Code = sorterCode.Replace(sorterCode, "MySorter")
},
RaftIdGenerator.NewId());
using (var json = context.ReadObject(command.ValueToJson(), command.Name))
{
ClusterStateMachine.PutValueDirectly(context, command.Name, json, 1);
}
tx.Commit();
}
}
SorterCompilationCache.Instance.RemoveServerWideItem(sorterName);
using (var server = GetNewServer(new ServerCreationOptions
{
DataDirectory = serverPath,
RunInMemory = false,
DeletePrevious = false
}))
using (var store = GetDocumentStore(new Options
{
ModifyDatabaseName = _ => sorterName,
Path = databasePath,
RunInMemory = false,
Server = server,
CreateDatabase = false
}))
{
CanUseSorterInternal <RavenException>(store, "is an implementation of a faulty sorter", "is an implementation of a faulty sorter", sorterName);
// can override with database sorter
store.Maintenance.Send(new PutSortersOperation(new SorterDefinition
{
Name = sorterName,
Code = GetSorter("RavenDB_8355.MySorter.cs", "MySorter", sorterName)
}));
CanUseSorterInternal <RavenException>(store, "Catch me: Name:2:0:False", "Catch me: Name:2:0:True", sorterName);
// can go back to server analyzer
store.Maintenance.Send(new DeleteSorterOperation(sorterName));
CanUseSorterInternal <RavenException>(store, "is an implementation of a faulty sorter", "is an implementation of a faulty sorter", sorterName);
// can fix server sorter
store.Maintenance.Server.Send(new PutServerWideSortersOperation(new SorterDefinition
{
Name = sorterName,
Code = GetSorter("RavenDB_8355.MySorter.cs", "MySorter", sorterName)
}));
CanUseSorterInternal <RavenException>(store, "Catch me: Name:2:0:False", "Catch me: Name:2:0:True", sorterName);
}
}
private static async Task GetIdentitiesValues(JsonOperationContext ctx, DocumentDatabase database, ServerStore serverStore,
List <string> identities, List <int> positionInListToCommandIndex, CommandData[] cmds)
{
var newIds = await serverStore.GenerateClusterIdentitiesBatchAsync(database.Name, identities, RaftIdGenerator.NewId());
Debug.Assert(newIds.Count == identities.Count);
var emptyChangeVector = ctx.GetLazyString("");
for (var index = 0; index < positionInListToCommandIndex.Count; index++)
{
var value = positionInListToCommandIndex[index];
cmds[value].Id = cmds[value].Id.Substring(0, cmds[value].Id.Length - 1) + "/" + newIds[index];
if (string.IsNullOrEmpty(cmds[value].ChangeVector) == false)
{
ThrowInvalidUsageOfChangeVectorWithIdentities(cmds[value]);
}
cmds[value].ChangeVector = emptyChangeVector;
}
}
private async Task AddDatabases()
{
await _locker.WaitAsync();
try
{
using (_server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
context.OpenReadTransaction();
var databases = _server
.ServerStore
.Cluster
.ItemKeysStartingWith(context, Constants.Documents.Prefix, 0, int.MaxValue)
.Select(x => x.Substring(Constants.Documents.Prefix.Length))
.ToList();
if (databases.Count == 0)
{
return;
}
var mapping = GetMapping(_server.ServerStore, context);
var missingDatabases = new List <string>();
foreach (var database in databases)
{
if (mapping.ContainsKey(database) == false)
{
missingDatabases.Add(database);
}
}
if (missingDatabases.Count > 0)
{
context.CloseTransaction();
var result = await _server.ServerStore.SendToLeaderAsync(new UpdateSnmpDatabasesMappingCommand(missingDatabases, RaftIdGenerator.NewId()));
await _server.ServerStore.Cluster.WaitForIndexNotification(result.Index);
context.OpenReadTransaction();
mapping = GetMapping(_server.ServerStore, context);
}
foreach (var database in databases)
{
LoadDatabase(database, mapping[database]);
}
}
}
finally
{
_locker.Release();
}
}
public void CanUseCustomAnalyzer_Restart_Faulty()
{
var serverPath = NewDataPath();
var databasePath = NewDataPath();
IOExtensions.DeleteDirectory(serverPath);
IOExtensions.DeleteDirectory(databasePath);
var analyzerName = GetDatabaseName();
using (var server = GetNewServer(new ServerCreationOptions
{
DataDirectory = serverPath,
RunInMemory = false
}))
using (var store = GetDocumentStore(new Options
{
ModifyDatabaseName = _ => analyzerName,
Path = databasePath,
RunInMemory = false,
Server = server,
DeleteDatabaseOnDispose = false
}))
{
var e = Assert.Throws <IndexCompilationException>(() => store.ExecuteIndex(new MyIndex(analyzerName)));
Assert.Contains($"Cannot find analyzer type '{analyzerName}' for field: Name", e.Message);
var analyzerCode = GetAnalyzer("RavenDB_14939.MyAnalyzer.cs", "MyAnalyzer", analyzerName);
store.Maintenance.Server.Send(new PutServerWideAnalyzersOperation(new AnalyzerDefinition
{
Name = analyzerName,
Code = analyzerCode
}));
store.ExecuteIndex(new MyIndex(analyzerName));
Fill(store);
Indexes.WaitForIndexing(store);
AssertCount <MyIndex>(store);
server.ServerStore.DatabasesLandlord.UnloadDirectly(store.Database);
// skipping compilation on purpose
using (server.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext context))
using (var tx = context.OpenWriteTransaction())
{
var command = new PutServerWideAnalyzerCommand(
new AnalyzerDefinition {
Name = analyzerName, Code = analyzerCode.Replace(analyzerName, "MyAnalyzer")
},
RaftIdGenerator.NewId());
using (var json = context.ReadObject(command.ValueToJson(), command.Name))
{
ClusterStateMachine.PutValueDirectly(context, command.Name, json, 1);
}
tx.Commit();
}
}
AnalyzerCompilationCache.Instance.RemoveServerWideItem(analyzerName);
using (var server = GetNewServer(new ServerCreationOptions
{
DataDirectory = serverPath,
RunInMemory = false,
DeletePrevious = false
}))
using (var store = GetDocumentStore(new Options
{
ModifyDatabaseName = _ => analyzerName,
Path = databasePath,
RunInMemory = false,
Server = server,
CreateDatabase = false
}))
{
store.Maintenance.Send(new ResetIndexOperation(new MyIndex(analyzerName).IndexName));
AssertErrors(store);
// can override with database analyzer
store.Maintenance.Send(new PutAnalyzersOperation(new AnalyzerDefinition
{
Name = analyzerName,
Code = GetAnalyzer("RavenDB_14939.MyAnalyzer.cs", "MyAnalyzer", analyzerName)
}));
store.Maintenance.Send(new ResetIndexOperation(new MyIndex(analyzerName).IndexName));
Indexes.WaitForIndexing(store);
AssertCount <MyIndex>(store);
// can go back to server analyzer
store.Maintenance.Send(new DeleteAnalyzerOperation(analyzerName));
store.Maintenance.Send(new ResetIndexOperation(new MyIndex(analyzerName).IndexName));
AssertErrors(store);
// can fix server analyzer
store.Maintenance.Server.Send(new PutServerWideAnalyzersOperation(new AnalyzerDefinition
{
Name = analyzerName,
Code = GetAnalyzer("RavenDB_14939.MyAnalyzer.cs", "MyAnalyzer", analyzerName)
}));
store.Maintenance.Send(new ResetIndexOperation(new MyIndex(analyzerName).IndexName));
Indexes.WaitForIndexing(store);
AssertCount <MyIndex>(store);
}
void AssertErrors(IDocumentStore store)
{
var errors = Indexes.WaitForIndexingErrors(store);
Assert.Equal(1, errors.Length);
Assert.Equal(1, errors[0].Errors.Length);
Assert.Contains("is an implementation of a faulty analyzer", errors[0].Errors[0].Error);
Assert.Contains("Could not find type", errors[0].Errors[0].Error);
}
}
public async Task RemoveEntryFromRaftLogEP()
{
var(_, leader) = await CreateRaftCluster(3, watcherCluster : true);
var database = GetDatabaseName();
await CreateDatabaseInClusterInner(new DatabaseRecord(database), 3, leader.WebUrl, null);
using (var store = new DocumentStore {
Database = database, Urls = new[] { leader.WebUrl }
}.Initialize())
{
leader.ServerStore.Engine.StateMachine.Validator = new TestCommandValidator();
var cmd = new RachisConsensusTestBase.TestCommandWithRaftId("test", RaftIdGenerator.NewId());
await Assert.ThrowsAsync <UnknownClusterCommand>(() => leader.ServerStore.SendToLeaderAsync(cmd));
var cmd2 = new CreateDatabaseOperation.CreateDatabaseCommand(new DatabaseRecord("Toli"), 1);
_ = leader.ServerStore.SendToLeaderAsync(new AddDatabaseCommand(Guid.NewGuid().ToString())
{
Record = new DatabaseRecord("Toli")
{
Topology = new DatabaseTopology
{
Members = new List <string> {
"A", "B", "C"
},
Rehabs = new List <string> {
},
ReplicationFactor = 3
}
},
Name = "Toli"
});
foreach (var server in Servers)
{
Assert.False(server.ServerStore.DatabasesLandlord.IsDatabaseLoaded("Toli"));
}
var index = Cluster.LastRaftIndexForCommand(leader, nameof(TestCommandWithRaftId));
List <string> nodelist = new List <string>();
var res = await WaitForValueAsync(async() =>
{
nodelist = await store.Maintenance.SendAsync(new RemoveEntryFromRaftLogOperation(index));
return(nodelist.Count);
}, 3);
Assert.Equal(3, res);
long index2 = 0;
foreach (var server in Servers)
{
await WaitForValueAsync(async() =>
{
var documentDatabase = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database);
using (documentDatabase.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext context))
using (var tx = context.OpenReadTransaction())
{
server.ServerStore.Engine.GetLastCommitIndex(context, out index2, out long term);
}
return(index2 > index);
}, true);
}
Assert.True(index2 > index, $"State machine is stuck. raft index was {index}, after remove raft entry index is {index2} ");
foreach (var server in Servers)
{
var val = WaitForValueAsync(() => server.ServerStore.DatabasesLandlord.IsDatabaseLoaded("Toli"), true);
Assert.True(val.Result);
Assert.Contains(server.ServerStore.NodeTag, nodelist);
}
}
}
public async Task SessionInSubscriptionsShouldNotTrackRevisions()
{
using (var store = GetDocumentStore())
{
var subscriptionId = await store.Subscriptions.CreateAsync <Revision <User> >();
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var configuration = new RevisionsConfiguration
{
Default = new RevisionsCollectionConfiguration
{
Disabled = false,
MinimumRevisionsToKeep = 5,
},
Collections = new Dictionary <string, RevisionsCollectionConfiguration>
{
["Users"] = new RevisionsCollectionConfiguration
{
Disabled = false
}
}
};
await Server
.ServerStore
.ModifyDatabaseRevisions(
context,
store.Database,
EntityToBlittable.ConvertCommandToBlittable(configuration, context), RaftIdGenerator.NewId());
}
for (int i = 0; i < 10; i++)
{
for (var j = 0; j < 10; j++)
{
using (var session = store.OpenSession())
{
session.Store(new User
{
Name = $"users{i} ver {j}"
}, "users/" + i);
session.SaveChanges();
}
}
}
using (var sub = store.Subscriptions.GetSubscriptionWorker <Revision <User> >(new SubscriptionWorkerOptions(subscriptionId)
{
TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5)
}))
{
Exception exception = null;
var mre = new AsyncManualResetEvent();
GC.KeepAlive(sub.Run(x =>
{
try
{
using (var session = x.OpenSession())
{
x.Items[0].Result.Current.Name = "aaaa";
session.SaveChanges();
}
}
catch (Exception e)
{
exception = e;
}
finally
{
mre.Set();
}
}));
Assert.True(await mre.WaitAsync(_reasonableWaitTime));
if (exception != null)
{
throw exception;
}
}
}
}
public async Task RemoveEntryFromRaftLogTest()
{
var(_, leader) = await CreateRaftCluster(3, watcherCluster : true);
var database = GetDatabaseName();
await CreateDatabaseInClusterInner(new DatabaseRecord(database), 3, leader.WebUrl, null);
using (var store = new DocumentStore
{
Database = database,
Urls = new[] { leader.WebUrl }
}.Initialize())
{
await ActionWithLeader(l => l.ServerStore.Engine.StateMachine.Validator = new TestCommandValidator());
var documentDatabase = await leader.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database);
var cmd = new RachisConsensusTestBase.TestCommandWithRaftId("test", RaftIdGenerator.NewId());
_ = leader.ServerStore.Engine.CurrentLeader.PutAsync(cmd, new TimeSpan(2000));
var cmd2 = new CreateDatabaseOperation.CreateDatabaseCommand(new DatabaseRecord("Toli"), 1);
_ = leader.ServerStore.SendToLeaderAsync(new AddDatabaseCommand(Guid.NewGuid().ToString())
{
Record = new DatabaseRecord("Toli")
{
Topology = new DatabaseTopology
{
Members = new List <string> {
"A", "B", "C"
},
Rehabs = new List <string> {
},
ReplicationFactor = 3
}
},
Name = "Toli"
});
foreach (var server in Servers)
{
Assert.False(server.ServerStore.DatabasesLandlord.IsDatabaseLoaded("Toli"));
}
long index = 0;
foreach (var server in Servers)
{
index = 0;
documentDatabase = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database);
using (documentDatabase.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext context))
using (var tx = context.OpenReadTransaction())
{
server.ServerStore.Engine.GetLastCommitIndex(context, out index, out long term);
}
var res = await WaitForValueAsync(() => server.ServerStore.Engine.RemoveEntryFromRaftLog(index + 1), true);
Assert.True(res);
}
long index2 = 0;
foreach (var server in Servers)
{
await WaitForValueAsync(async() =>
{
index2 = 0;
documentDatabase = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database);
using (documentDatabase.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext context))
using (var tx = context.OpenReadTransaction())
{
server.ServerStore.Engine.GetLastCommitIndex(context, out index2, out long term);
}
return(index2 > index);
}, true);
}
Assert.True(index2 > index, $"State machine is stuck. raft index was {index}, after remove raft entry index is {index2} ");
foreach (var server in Servers)
{
var val = await WaitForValueAsync(() => server.ServerStore.DatabasesLandlord.IsDatabaseLoaded("Toli"), true);
Assert.True(val);
}
}
}
public void Cannot_Setup_Documents_Compression_When_Experimental_Mode_Is_Off()
{
DoNotReuseServer(new Dictionary <string, string>
{
{ RavenConfiguration.GetKey(x => x.Core.FeaturesAvailability), FeaturesAvailability.Stable.ToString() }
});
using (var store = GetDocumentStore())
{
var databaseRecord = store.Maintenance.Server.Send(new GetDatabaseRecordOperation(store.Database));
Assert.Null(databaseRecord.DocumentsCompression);
var databaseName = $"{store.Database}-{Guid.NewGuid()}";
using (EnsureDatabaseDeletion(databaseName, store))
{
var e = Assert.Throws <RavenException>(() => store.Maintenance.Server.Send(new CreateDatabaseOperation(new DatabaseRecord
{
DatabaseName = databaseName,
DocumentsCompression = new DocumentsCompressionConfiguration
{
CompressRevisions = true
}
})));
Assert.Contains("Can not use 'Documents Compression'", e.Message);
store.Maintenance.Server.Send(new CreateDatabaseOperation(new DatabaseRecord
{
DatabaseName = databaseName
}));
using (var commands = store.Commands(databaseName))
{
e = Assert.Throws <RavenException>(() => commands.ExecuteJson($"/admin/documents-compression/config?raft-request-id={RaftIdGenerator.NewId()}", HttpMethod.Post, new DocumentsCompressionConfiguration
{
CompressRevisions = true
}));
databaseRecord = store.Maintenance.Server.Send(new GetDatabaseRecordOperation(databaseName));
Assert.Null(databaseRecord.DocumentsCompression);
}
}
}
}
public async Task <IOperationResult> Execute(Action <IOperationProgress> onProgress)
{
var databaseName = RestoreFromConfiguration.DatabaseName;
var result = new RestoreResult
{
DataDirectory = RestoreFromConfiguration.DataDirectory
};
try
{
var filesToRestore = await GetOrderedFilesToRestore();
using (_serverStore.ContextPool.AllocateOperationContext(out JsonOperationContext serverContext))
{
if (onProgress == null)
{
onProgress = _ => { }
}
;
Stopwatch sw = null;
RestoreSettings restoreSettings = null;
var firstFile = filesToRestore[0];
var extension = Path.GetExtension(firstFile);
var snapshotRestore = false;
if ((extension == Constants.Documents.PeriodicBackup.SnapshotExtension) ||
(extension == Constants.Documents.PeriodicBackup.EncryptedSnapshotExtension))
{
onProgress.Invoke(result.Progress);
snapshotRestore = true;
sw = Stopwatch.StartNew();
if (extension == Constants.Documents.PeriodicBackup.EncryptedSnapshotExtension)
{
_hasEncryptionKey = RestoreFromConfiguration.EncryptionKey != null ||
RestoreFromConfiguration.BackupEncryptionSettings?.Key != null;
}
// restore the snapshot
restoreSettings = await SnapshotRestore(serverContext, firstFile, onProgress, result);
if (restoreSettings != null && RestoreFromConfiguration.SkipIndexes)
{
// remove all indexes from the database record
restoreSettings.DatabaseRecord.AutoIndexes = null;
restoreSettings.DatabaseRecord.Indexes = null;
}
// removing the snapshot from the list of files
filesToRestore.RemoveAt(0);
}
else
{
result.SnapshotRestore.Skipped = true;
result.SnapshotRestore.Processed = true;
onProgress.Invoke(result.Progress);
}
if (restoreSettings == null)
{
restoreSettings = new RestoreSettings
{
DatabaseRecord = new DatabaseRecord(databaseName)
{
// we only have a smuggler restore
// use the encryption key to encrypt the database
Encrypted = _hasEncryptionKey
}
};
DatabaseHelper.Validate(databaseName, restoreSettings.DatabaseRecord, _serverStore.Configuration);
}
var databaseRecord = restoreSettings.DatabaseRecord;
if (databaseRecord.Settings == null)
{
databaseRecord.Settings = new Dictionary <string, string>();
}
var runInMemoryConfigurationKey = RavenConfiguration.GetKey(x => x.Core.RunInMemory);
databaseRecord.Settings.Remove(runInMemoryConfigurationKey);
if (_serverStore.Configuration.Core.RunInMemory)
{
databaseRecord.Settings[runInMemoryConfigurationKey] = "false";
}
var dataDirectoryConfigurationKey = RavenConfiguration.GetKey(x => x.Core.DataDirectory);
databaseRecord.Settings.Remove(dataDirectoryConfigurationKey); // removing because we want to restore to given location, not to serialized in backup one
if (_restoringToDefaultDataDirectory == false)
{
databaseRecord.Settings[dataDirectoryConfigurationKey] = RestoreFromConfiguration.DataDirectory;
}
if (_hasEncryptionKey)
{
// save the encryption key so we'll be able to access the database
_serverStore.PutSecretKey(RestoreFromConfiguration.EncryptionKey,
databaseName, overwrite: false);
}
var addToInitLog = new Action <string>(txt => // init log is not save in mem during RestoreBackup
{
var msg = $"[RestoreBackup] {DateTime.UtcNow} :: Database '{databaseName}' : {txt}";
if (Logger.IsInfoEnabled)
{
Logger.Info(msg);
}
});
var configuration = _serverStore
.DatabasesLandlord
.CreateDatabaseConfiguration(databaseName, ignoreDisabledDatabase: true, ignoreBeenDeleted: true, ignoreNotRelevant: true, databaseRecord);
using (var database = new DocumentDatabase(databaseName, configuration, _serverStore, addToInitLog))
{
// smuggler needs an existing document database to operate
var options = InitializeOptions.SkipLoadingDatabaseRecord;
if (snapshotRestore)
{
options |= InitializeOptions.GenerateNewDatabaseId;
}
database.Initialize(options);
databaseRecord.Topology = new DatabaseTopology();
// restoring to the current node only
databaseRecord.Topology.Members.Add(_nodeTag);
// we are currently restoring, shouldn't try to access it
databaseRecord.DatabaseState = DatabaseStateStatus.RestoreInProgress;
var(index, _) = await _serverStore.WriteDatabaseRecordAsync(databaseName, databaseRecord, null, RaftIdGenerator.NewId(), restoreSettings.DatabaseValues, isRestore : true);
await _serverStore.Cluster.WaitForIndexNotification(index);
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
if (snapshotRestore)
{
await RestoreFromSmugglerFile(onProgress, database, firstFile, context);
await SmugglerRestore(database, filesToRestore, context, databaseRecord, onProgress, result);
result.SnapshotRestore.Processed = true;
var summary = database.GetDatabaseSummary();
result.Documents.ReadCount += summary.DocumentsCount;
result.Documents.Attachments.ReadCount += summary.AttachmentsCount;
result.Counters.ReadCount += summary.CounterEntriesCount;
result.RevisionDocuments.ReadCount += summary.RevisionsCount;
result.Conflicts.ReadCount += summary.ConflictsCount;
result.Indexes.ReadCount += databaseRecord.GetIndexesCount();
result.CompareExchange.ReadCount += summary.CompareExchangeCount;
result.CompareExchangeTombstones.ReadCount += summary.CompareExchangeTombstonesCount;
result.Identities.ReadCount += summary.IdentitiesCount;
result.AddInfo($"Successfully restored {result.SnapshotRestore.ReadCount} files during snapshot restore, took: {sw.ElapsedMilliseconds:#,#;;0}ms");
onProgress.Invoke(result.Progress);
}
else
{
await SmugglerRestore(database, filesToRestore, context, databaseRecord, onProgress, result);
}
DisableOngoingTasksIfNeeded(databaseRecord);
result.DatabaseRecord.Processed = true;
result.Documents.Processed = true;
result.RevisionDocuments.Processed = true;
result.Conflicts.Processed = true;
result.Indexes.Processed = true;
result.Counters.Processed = true;
result.Identities.Processed = true;
result.CompareExchange.Processed = true;
result.Subscriptions.Processed = true;
onProgress.Invoke(result.Progress);
}
}
// after the db for restore is done, we can safely set the db state to normal and write the DatabaseRecord
databaseRecord.DatabaseState = DatabaseStateStatus.Normal;
var(updateIndex, _) = await _serverStore.WriteDatabaseRecordAsync(databaseName, databaseRecord, null, RaftIdGenerator.DontCareId, isRestore : true);
await _serverStore.Cluster.WaitForIndexNotification(updateIndex);
if (databaseRecord.Topology.RelevantFor(_serverStore.NodeTag))
{
// we need to wait for the database record change to be propagated properly
var db = await _serverStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName);
await db.RachisLogIndexNotifications.WaitForIndexNotification(updateIndex, _operationCancelToken.Token);
}
return(result);
}
}
catch (Exception e)
{
if (Logger.IsOperationsEnabled)
{
Logger.Operations("Failed to restore database", e);
}
var alert = AlertRaised.Create(
RestoreFromConfiguration.DatabaseName,
"Failed to restore database",
$"Could not restore database named {RestoreFromConfiguration.DatabaseName}",
AlertType.RestoreError,
NotificationSeverity.Error,
details: new ExceptionDetails(e));
_serverStore.NotificationCenter.Add(alert);
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
bool databaseExists;
using (context.OpenReadTransaction())
{
databaseExists = _serverStore.Cluster.DatabaseExists(context, RestoreFromConfiguration.DatabaseName);
}
if (databaseExists == false)
{
// delete any files that we already created during the restore
IOExtensions.DeleteDirectory(RestoreFromConfiguration.DataDirectory);
}
else
{
var deleteResult = await _serverStore.DeleteDatabaseAsync(RestoreFromConfiguration.DatabaseName, true, new[] { _serverStore.NodeTag }, RaftIdGenerator.DontCareId);
await _serverStore.Cluster.WaitForIndexNotification(deleteResult.Index);
}
}
result.AddError($"Error occurred during restore of database {databaseName}. Exception: {e.Message}");
onProgress.Invoke(result.Progress);
throw;
}
finally
{
Dispose();
}
}
private async Task AddDatabases()
{
await _locker.WaitAsync();
List <string> databases = null;
List <string> missingDatabases = null;
try
{
using (_server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
context.OpenReadTransaction();
databases = _server
.ServerStore
.Cluster
.ItemKeysStartingWith(context, Constants.Documents.Prefix, 0, int.MaxValue)
.Select(x => x.Substring(Constants.Documents.Prefix.Length))
.ToList();
if (databases.Count == 0)
{
return;
}
var mapping = GetMapping(_server.ServerStore, context);
missingDatabases = new List <string>();
foreach (var database in databases)
{
if (mapping.ContainsKey(database) == false)
{
missingDatabases.Add(database);
}
}
if (missingDatabases.Count > 0)
{
context.CloseTransaction();
try
{
var result = await _server.ServerStore.SendToLeaderAsync(new UpdateSnmpDatabasesMappingCommand(missingDatabases, RaftIdGenerator.NewId()));
await _server.ServerStore.Cluster.WaitForIndexNotification(result.Index);
}
catch (Exception e) when(e is TimeoutException || e is OperationCanceledException)
{
// we will update it in the OnDatabaseLoaded event
return;
}
context.OpenReadTransaction();
mapping = GetMapping(_server.ServerStore, context);
}
foreach (var database in databases)
{
LoadDatabase(database, mapping[database]);
}
}
}
catch (Exception e)
{
var msg = "Failed to update the SNMP mapping";
if (databases?.Count > 0)
{
msg += $" databases to update: {string.Join(", ", databases)}";
}
if (missingDatabases?.Count > 0)
{
msg += $" missing databases: {string.Join(", ", missingDatabases)}";
}
if (Logger.IsOperationsEnabled)
{
Logger.Operations(msg, e);
}
}
finally
{
_locker.Release();
}
}
private async Task CreateDatabaseIfNeeded(string databaseName)
{
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
using (var rawRecord = _serverStore.Cluster.ReadRawDatabaseRecord(context, databaseName))
{
if (rawRecord != null)
{
if (rawRecord.Topology.AllNodes.Contains(_serverStore.NodeTag) == false)
{
throw new InvalidOperationException(
"Cannot migrate database because " +
$"database doesn't exist on the current node ({_serverStore.NodeTag})");
}
// database already exists
return;
}
}
var databaseRecord = new DatabaseRecord(databaseName)
{
Topology = new DatabaseTopology
{
Members =
{
_serverStore.NodeTag
}
}
};
var(index, _) = await _serverStore.WriteDatabaseRecordAsync(databaseName, databaseRecord, null, RaftIdGenerator.NewId());
await _serverStore.Cluster.WaitForIndexNotification(index);
}
public static void SaveBackupStatus(PeriodicBackupStatus status, DocumentDatabase documentDatabase, Logger logger, BackupResult backupResult)
{
try
{
var command = new UpdatePeriodicBackupStatusCommand(documentDatabase.Name, RaftIdGenerator.NewId())
{
PeriodicBackupStatus = status
};
var result = AsyncHelpers.RunSync(() => documentDatabase.ServerStore.SendToLeaderAsync(command));
AsyncHelpers.RunSync(() => documentDatabase.ServerStore.WaitForCommitIndexChange(RachisConsensus.CommitIndexModification.GreaterOrEqual, result.Index));
if (logger.IsInfoEnabled)
{
logger.Info($"Periodic backup status with task id {status.TaskId} was updated");
}
}
catch (Exception e)
{
const string message = "Error saving the periodic backup status";
if (logger.IsOperationsEnabled)
{
logger.Operations(message, e);
}
backupResult?.AddError($"{message}{Environment.NewLine}{e}");
}
}
public async Task CanReplaceClusterCertWithExtensionPoint()
{
string loadAndRenewScript;
string certChangedScript;
IDictionary <string, string> customSettings1 = new ConcurrentDictionary <string, string>();
IDictionary <string, string> customSettings2 = new ConcurrentDictionary <string, string>();
IDictionary <string, string> customSettings3 = new ConcurrentDictionary <string, string>();
var certPath = GenerateAndSaveSelfSignedCertificate();
var cert2Path = GenerateAndSaveSelfSignedCertificate(createNew: true);
var outputFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".txt"));
var firstServerCert = new X509Certificate2(certPath, (string)null, X509KeyStorageFlags.Exportable | X509KeyStorageFlags.MachineKeySet);
var newServerCert = new X509Certificate2(cert2Path, (string)null, X509KeyStorageFlags.Exportable | X509KeyStorageFlags.MachineKeySet);
if (PlatformDetails.RunningOnPosix)
{
var loadAndRenewScriptNode1Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".sh"));
var loadAndRenewScriptNode2Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".sh"));
var loadAndRenewScriptNode3Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".sh"));
var certChangedScriptNode1Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".sh"));
var certChangedScriptNode2Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".sh"));
var certChangedScriptNode3Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".sh"));
var loadArgsNode1 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
loadAndRenewScriptNode1Path, certPath
});
var renewArgsNode1 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
loadAndRenewScriptNode1Path, cert2Path
});
var certChangedArgsNode1 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
certChangedScriptNode1Path, outputFile
});
var loadArgsNode2 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
loadAndRenewScriptNode2Path, certPath
});
var renewArgsNode2 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
loadAndRenewScriptNode2Path, cert2Path
});
var certChangedArgsNode2 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
certChangedScriptNode2Path, outputFile
});
var loadArgsNode3 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
loadAndRenewScriptNode3Path, certPath
});
var renewArgsNode3 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
loadAndRenewScriptNode3Path, cert2Path
});
var certChangedArgsNode3 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
certChangedScriptNode3Path, outputFile
});
customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExec)] = "bash";
customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExecArguments)] = $"{loadArgsNode1}";
customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExec)] = "bash";
customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExecArguments)] = $"{renewArgsNode1}";
customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExec)] = "bash";
customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExecArguments)] = $"{certChangedArgsNode1}";
customSettings1[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = "https://" + Environment.MachineName + ":0";
customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExec)] = "bash";
customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExecArguments)] = $"{loadArgsNode2}";
customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExec)] = "bash";
customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExecArguments)] = $"{renewArgsNode2}";
customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExec)] = "bash";
customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExecArguments)] = $"{certChangedArgsNode2}";
customSettings2[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = "https://" + Environment.MachineName + ":0";
customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExec)] = "bash";
customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExecArguments)] = $"{loadArgsNode3}";
customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExec)] = "bash";
customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExecArguments)] = $"{renewArgsNode3}";
customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExec)] = "bash";
customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExecArguments)] = $"{certChangedArgsNode3}";
customSettings3[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = "https://" + Environment.MachineName + ":0";
loadAndRenewScript = "#!/bin/bash\ncat -u \"$1\"";
certChangedScript = "#!/bin/bash\nwhile read line; do\n\techo \"$line\" >> \"$1\"\ndone < /dev/stdin";
File.WriteAllText(loadAndRenewScriptNode1Path, loadAndRenewScript);
File.WriteAllText(loadAndRenewScriptNode2Path, loadAndRenewScript);
File.WriteAllText(loadAndRenewScriptNode3Path, loadAndRenewScript);
File.WriteAllText(certChangedScriptNode1Path, certChangedScript);
File.WriteAllText(certChangedScriptNode2Path, certChangedScript);
File.WriteAllText(certChangedScriptNode3Path, certChangedScript);
Process.Start("chmod", $"700 {loadAndRenewScriptNode1Path}");
Process.Start("chmod", $"700 {loadAndRenewScriptNode2Path}");
Process.Start("chmod", $"700 {loadAndRenewScriptNode3Path}");
Process.Start("chmod", $"700 {certChangedScriptNode1Path}");
Process.Start("chmod", $"700 {certChangedScriptNode2Path}");
Process.Start("chmod", $"700 {certChangedScriptNode3Path}");
}
else
{
var loadAndRenewScriptNode1Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".ps1"));
var loadAndRenewScriptNode2Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".ps1"));
var loadAndRenewScriptNode3Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".ps1"));
var certChangedScriptNode1Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".ps1"));
var certChangedScriptNode2Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".ps1"));
var certChangedScriptNode3Path = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".ps1"));
var loadArgsNode1 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
"-NoProfile", loadAndRenewScriptNode1Path, certPath
});
var renewArgsNode1 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
"-NoProfile", loadAndRenewScriptNode1Path, cert2Path
});
var certChangedArgsNode1 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
"-NoProfile", certChangedScriptNode1Path, outputFile
});
var loadArgsNode2 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
"-NoProfile", loadAndRenewScriptNode2Path, certPath
});
var renewArgsNode2 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
"-NoProfile", loadAndRenewScriptNode2Path, cert2Path
});
var certChangedArgsNode2 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
"-NoProfile", certChangedScriptNode2Path, outputFile
});
var loadArgsNode3 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
"-NoProfile", loadAndRenewScriptNode3Path, certPath
});
var renewArgsNode3 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
"-NoProfile", loadAndRenewScriptNode3Path, cert2Path
});
var certChangedArgsNode3 = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
"-NoProfile", certChangedScriptNode3Path, outputFile
});
customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExec)] = "powershell";
customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExecArguments)] = $"{loadArgsNode1}";
customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExec)] = "powershell";
customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExecArguments)] = $"{renewArgsNode1}";
customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExec)] = "powershell";
customSettings1[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExecArguments)] = $"{certChangedArgsNode1}";
customSettings1[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = "https://" + Environment.MachineName + ":0";
customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExec)] = "powershell";
customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExecArguments)] = $"{loadArgsNode2}";
customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExec)] = "powershell";
customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExecArguments)] = $"{renewArgsNode2}";
customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExec)] = "powershell";
customSettings2[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExecArguments)] = $"{certChangedArgsNode2}";
customSettings2[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = "https://" + Environment.MachineName + ":0";
customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExec)] = "powershell";
customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExecArguments)] = $"{loadArgsNode3}";
customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExec)] = "powershell";
customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExecArguments)] = $"{renewArgsNode3}";
customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExec)] = "powershell";
customSettings3[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExecArguments)] = $"{certChangedArgsNode3}";
customSettings3[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = "https://" + Environment.MachineName + ":0";
loadAndRenewScript = @"param([string]$userArg1)
try {
$bytes = Get-Content -path $userArg1 -encoding Byte
$stdout = [System.Console]::OpenStandardOutput()
$stdout.Write($bytes, 0, $bytes.Length)
}
catch {
Write-Error $_.Exception
exit 1
}
exit 0";
certChangedScript = @"param([string]$userArg1)
try {
$certBase64 = [System.Console]::ReadLine()
Add-Content $userArg1 $certBase64
}
catch {
Write-Error $_.Exception
exit 1
}
exit 0";
File.WriteAllText(loadAndRenewScriptNode1Path, loadAndRenewScript);
File.WriteAllText(loadAndRenewScriptNode2Path, loadAndRenewScript);
File.WriteAllText(loadAndRenewScriptNode3Path, loadAndRenewScript);
File.WriteAllText(certChangedScriptNode1Path, certChangedScript);
File.WriteAllText(certChangedScriptNode2Path, certChangedScript);
File.WriteAllText(certChangedScriptNode3Path, certChangedScript);
}
var clusterSize = 3;
var databaseName = GetDatabaseName();
var leader = await CreateRaftClusterAndGetLeader(clusterSize, false, useSsl : true, customSettingsList : new List <IDictionary <string, string> > {
customSettings1, customSettings2, customSettings3
});
Assert.True(leader?.Certificate?.Certificate?.Thumbprint?.Equals(firstServerCert.Thumbprint), "Cert is identical");
var adminCertificate = AskServerForClientCertificate(certPath, new Dictionary <string, DatabaseAccess>(), SecurityClearance.ClusterAdmin, server: leader);
DatabasePutResult databaseResult;
using (var store = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName,
Certificate = adminCertificate,
Conventions =
{
DisableTopologyUpdates = true
}
}.Initialize())
{
var doc = new DatabaseRecord(databaseName);
databaseResult = await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(doc, clusterSize));
}
Assert.Equal(clusterSize, databaseResult.Topology.AllNodes.Count());
foreach (var server in Servers)
{
await server.ServerStore.Cluster.WaitForIndexNotification(databaseResult.RaftCommandIndex);
}
foreach (var server in Servers.Where(s => databaseResult.NodesAddedTo.Any(n => n == s.WebUrl)))
{
await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName);
}
using (var store = new DocumentStore()
{
Urls = new[] { databaseResult.NodesAddedTo[0] },
Database = databaseName,
Certificate = adminCertificate,
Conventions =
{
DisableTopologyUpdates = true
}
}.Initialize())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "Karmelush" }, "users/1");
await session.SaveChangesAsync();
}
var mre = new ManualResetEventSlim();
leader.ServerCertificateChanged += (sender, args) => mre.Set();
// This will initiate the refresh cycle, and ask a new certificate from the executable
leader.RefreshClusterCertificate(false, RaftIdGenerator.NewId());
Assert.True(mre.Wait(Debugger.IsAttached ? TimeSpan.FromMinutes(10) : TimeSpan.FromMinutes(2)), "Waited too long");
Assert.NotNull(leader.Certificate.Certificate.Thumbprint);
Assert.True(leader.Certificate.Certificate.Thumbprint.Equals(newServerCert.Thumbprint), "New cert is identical");
var base64NewCertWrittenByExecutable = File.ReadAllLines(outputFile)[0];
var loadedCertificate = new X509Certificate2(Convert.FromBase64String(base64NewCertWrittenByExecutable));
Assert.Equal(newServerCert.Thumbprint, loadedCertificate.Thumbprint);
using (var session = store.OpenSession())
{
var user1 = session.Load <User>("users/1");
Assert.NotNull(user1);
Assert.Equal("Karmelush", user1.Name);
}
}
}
