public async Task UpdatePullReplicationOnSinkNode()
{
if (ResourceNameValidator.IsValidResourceName(Database.Name, ServerStore.Configuration.Core.DataDirectory.FullPath, out string errorMessage) == false)
{
throw new BadRequestException(errorMessage);
}
ServerStore.LicenseManager.AssertCanAddPullReplicationAsSink();
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
PullReplicationAsSink pullReplication = null;
await DatabaseConfigurations(
(_, databaseName, blittableJson, guid) => ServerStore.UpdatePullReplicationAsSink(databaseName, blittableJson, guid, out pullReplication),
"update-sink-pull-replication", GetRaftRequestIdFromQuery(),
fillJson : (json, _, index) =>
{
using (context.OpenReadTransaction())
{
var topology = ServerStore.Cluster.ReadDatabaseTopology(context, Database.Name);
json[nameof(OngoingTask.ResponsibleNode)] = Database.WhoseTaskIsIt(topology, pullReplication, null);
}
json[nameof(ModifyOngoingTaskResult.TaskId)] = pullReplication.TaskId == 0 ? index : pullReplication.TaskId;
}, statusCode : HttpStatusCode.Created);
}
}
public async Task EnsureConnectionStringNameCantBeNull()
{
using (var store = GetDocumentStore())
{
var pull = new PullReplicationAsSink(store.Database, "test", "dummy");
pull.ConnectionStringName = null;
var op = new UpdatePullReplicationAsSinkOperation(pull);
var ex = await Assert.ThrowsAsync <RavenException>(async() => await store.Maintenance.SendAsync(op));
RavenTestHelper.AssertStartsWithRespectingNewLines("Raven.Server.Rachis.RachisApplyException: Failed to update database record.\r\n ---> System.ArgumentNullException: Value cannot be null.", ex.Message);
}
}
public async Task PullReplicationWithoutPrivateKey()
{
var hubSettings = new ConcurrentDictionary <string, string>();
var sinkSettings = new ConcurrentDictionary <string, string>();
var hubCertificates = GenerateAndSaveSelfSignedCertificate(createNew: true);
var hubCerts = SetupServerAuthentication(hubSettings, certificates: hubCertificates);
var sinkCertificates = GenerateAndSaveSelfSignedCertificate(createNew: true);
var sinkCerts = SetupServerAuthentication(sinkSettings, certificates: sinkCertificates);
var hubDB = GetDatabaseName();
var sinkDB = GetDatabaseName();
var pullReplicationName = $"{hubDB}-pull";
var hubServer = GetNewServer(new ServerCreationOptions {
CustomSettings = hubSettings, RegisterForDisposal = true
});
var sinkServer = GetNewServer(new ServerCreationOptions {
CustomSettings = sinkSettings, RegisterForDisposal = true
});
var dummy = GenerateAndSaveSelfSignedCertificate(createNew: true);
var pullReplicationCertificate = new X509Certificate2(dummy.ServerCertificatePath, (string)null, X509KeyStorageFlags.MachineKeySet | X509KeyStorageFlags.Exportable);
Assert.True(pullReplicationCertificate.HasPrivateKey);
using (var hubStore = GetDocumentStore(new Options
{
ClientCertificate = hubCerts.ServerCertificate.Value,
Server = hubServer,
ModifyDatabaseName = _ => hubDB
}))
using (var sinkStore = GetDocumentStore(new Options
{
ClientCertificate = sinkCerts.ServerCertificate.Value,
Server = sinkServer,
ModifyDatabaseName = _ => sinkDB
}))
{
var pull = new PullReplicationAsSink(hubStore.Database, $"ConnectionString-{hubStore.Database}", pullReplicationName);
pull.CertificateWithPrivateKey = Convert.ToBase64String(pullReplicationCertificate.Export(X509ContentType.Cert));
await Assert.ThrowsAsync <AuthorizationException>(async() => await sinkStore.Maintenance.SendAsync(new UpdatePullReplicationAsSinkOperation(pull)));
}
}
public async Task DisablePullReplicationOnSink()
{
var definitionName = $"pull-replication {GetDatabaseName()}";
var timeout = 3000;
using (var sink = GetDocumentStore())
using (var hub = GetDocumentStore())
{
await hub.Maintenance.ForDatabase(hub.Database).SendAsync(new PutPullReplicationAsHubOperation(definitionName));
using (var main = hub.OpenSession())
{
main.Store(new User(), "hub/1");
main.SaveChanges();
}
var pullTasks = await SetupPullReplicationAsync(definitionName, sink, hub);
Assert.True(WaitForDocument(sink, "hub/1", timeout), sink.Identifier);
var pull = new PullReplicationAsSink(hub.Database, $"ConnectionString-{sink.Database}", definitionName)
{
Disabled = true,
TaskId = pullTasks[0].TaskId
};
await AddWatcherToReplicationTopology(sink, pull, hub.Urls);
using (var main = hub.OpenSession())
{
main.Store(new User(), "hub/2");
main.SaveChanges();
}
Assert.False(WaitForDocument(sink, "hub/2", timeout), sink.Identifier);
pull.Disabled = false;
await AddWatcherToReplicationTopology(sink, pull, hub.Urls);
using (var main = hub.OpenSession())
{
main.Store(new User(), "hub/3");
main.SaveChanges();
}
Assert.True(WaitForDocument(sink, "hub/2", timeout), sink.Identifier);
Assert.True(WaitForDocument(sink, "hub/3", timeout), sink.Identifier);
}
}
public async Task <List <ModifyOngoingTaskResult> > SetupPullReplicationAsync(string remoteName, DocumentStore sink, X509Certificate2 certificate, params DocumentStore[] hub)
{
var tasks = new List <Task <ModifyOngoingTaskResult> >();
var resList = new List <ModifyOngoingTaskResult>();
foreach (var store in hub)
{
var pull = new PullReplicationAsSink(store.Database, $"ConnectionString-{store.Database}", remoteName);
if (certificate != null)
{
pull.CertificateWithPrivateKey = Convert.ToBase64String(certificate.Export(X509ContentType.Pfx));
}
ModifyReplicationDestination(pull);
tasks.Add(AddWatcherToReplicationTopology(sink, pull, store.Urls));
}
await Task.WhenAll(tasks);
foreach (var task in tasks)
{
resList.Add(await task);
}
return(resList);
}
public async Task AutoNamingAlgorithmOfOngoingTasksShouldTakeNameAlreadyExistsIntoAccount()
{
using (var store = GetDocumentStore())
{
var dbName = $"db/{Guid.NewGuid()}";
var csName = $"cs/{Guid.NewGuid()}";
var connectionString = new RavenConnectionString
{
Name = csName,
Database = dbName,
TopologyDiscoveryUrls = new[] { "http://127.0.0.1:12345" }
};
var result = await store.Maintenance.SendAsync(new PutConnectionStringOperation <RavenConnectionString>(connectionString));
Assert.NotNull(result.RaftCommandIndex);
await store.Maintenance.SendAsync(new UpdateExternalReplicationOperation(new ExternalReplication(dbName, csName)));
await store.Maintenance.SendAsync(new UpdateExternalReplicationOperation(new ExternalReplication(dbName, csName)));
var backupConfig = new PeriodicBackupConfiguration
{
LocalSettings = new LocalSettings
{
FolderPath = NewDataPath(suffix: "BackupFolder")
},
AzureSettings = new AzureSettings
{
StorageContainer = "abc"
},
FullBackupFrequency = "* */1 * * *",
IncrementalBackupFrequency = "* */2 * * *",
Disabled = true
};
await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfig));
await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfig));
var etlConfiguration = new RavenEtlConfiguration
{
ConnectionStringName = csName,
Transforms =
{
new Transformation()
{
Name = "loadAll",
Collections ={ "Users" },
Script = "loadToUsers(this)"
}
}
};
await store.Maintenance.SendAsync(new AddEtlOperation <RavenConnectionString>(etlConfiguration));
await store.Maintenance.SendAsync(new AddEtlOperation <RavenConnectionString>(etlConfiguration));
// for Pull Replication Hub name is required - no need to test
var sink = new PullReplicationAsSink
{
HubDefinitionName = "aa",
ConnectionString = connectionString,
ConnectionStringName = connectionString.Name
};
await store.Maintenance.SendAsync(new UpdatePullReplicationAsSinkOperation(sink));
await store.Maintenance.SendAsync(new UpdatePullReplicationAsSinkOperation(sink));
}
}
public async Task FailoverOnSinkNodeFail()
{
var clusterSize = 3;
var hub = await CreateRaftClusterAndGetLeader(clusterSize);
var minion = await CreateRaftClusterAndGetLeader(clusterSize);
var hubDB = GetDatabaseName();
var minionDB = GetDatabaseName();
var dstTopology = await CreateDatabaseInCluster(minionDB, clusterSize, minion.WebUrl);
var srcTopology = await CreateDatabaseInCluster(hubDB, clusterSize, hub.WebUrl);
using (var hubStore = new DocumentStore
{
Urls = new[] { hub.WebUrl },
Database = hubDB
}.Initialize())
using (var minionStore = new DocumentStore
{
Urls = new[] { minion.WebUrl },
Database = minionDB
}.Initialize())
{
using (var session = hubStore.OpenSession())
{
session.Advanced.WaitForReplicationAfterSaveChanges(timeout: TimeSpan.FromSeconds(10), replicas: clusterSize - 1);
session.Store(new User
{
Name = "Karmel"
}, "users/1");
session.SaveChanges();
}
var name = $"pull-replication {GetDatabaseName()}";
await hubStore.Maintenance.ForDatabase(hubStore.Database).SendAsync(new PutPullReplicationAsHubOperation(name));
// add pull replication with invalid discovery url to test the failover on database topology discovery
var pullReplication = new PullReplicationAsSink(hubDB, $"ConnectionString-{hubDB}", name)
{
MentorNode = "B", // this is the node were the data will be replicated to.
};
await AddWatcherToReplicationTopology((DocumentStore)minionStore, pullReplication, new[] { "http://127.0.0.1:1234", hub.WebUrl });
using (var dstSession = minionStore.OpenSession())
{
Assert.True(await WaitForDocumentInClusterAsync <User>(
dstSession as DocumentSession,
"users/1",
u => u.Name.Equals("Karmel"),
TimeSpan.FromSeconds(30)));
}
var minionUrl = minion.ServerStore.GetClusterTopology().GetUrlFromTag("B");
var server = Servers.Single(s => s.WebUrl == minionUrl);
var handler = await InstantiateOutgoingTaskHandler(minionDB, server);
Assert.True(WaitForValue(
() => ((OngoingTaskPullReplicationAsSink)handler.GetOngoingTasksInternal().OngoingTasksList.Single(t => t is OngoingTaskPullReplicationAsSink)).DestinationUrl != null,
true));
// dispose the minion node.
DisposeServerAndWaitForFinishOfDisposal(server);
using (var session = hubStore.OpenSession())
{
session.Advanced.WaitForReplicationAfterSaveChanges(timeout: TimeSpan.FromSeconds(10), replicas: clusterSize - 2);
session.Store(new User
{
Name = "Karmel2"
}, "users/2");
session.SaveChanges();
}
var user = WaitForDocumentToReplicate <User>(
minionStore,
"users/2",
30_000);
Assert.Equal("Karmel2", user.Name);
}
}
public async Task RavenDB_15855()
{
DebuggerAttachedTimeout.DisableLongTimespan = true;
var clusterSize = 3;
var(_, hub) = await CreateRaftCluster(clusterSize);
var(minionNodes, minion) = await CreateRaftCluster(clusterSize);
var hubDB = GetDatabaseName();
var minionDB = GetDatabaseName();
var dstTopology = await CreateDatabaseInCluster(minionDB, clusterSize, minion.WebUrl);
var srcTopology = await CreateDatabaseInCluster(hubDB, clusterSize, hub.WebUrl);
using (var hubStore = new DocumentStore
{
Urls = new[] { hub.WebUrl },
Database = hubDB
}.Initialize())
using (var minionStore = new DocumentStore
{
Urls = new[] { minion.WebUrl },
Database = minionDB
}.Initialize())
{
using (var session = hubStore.OpenSession())
{
session.Advanced.WaitForReplicationAfterSaveChanges(timeout: TimeSpan.FromSeconds(10), replicas: clusterSize - 1);
session.Store(new User
{
Name = "Karmel"
}, "users/1");
session.SaveChanges();
}
var name = $"pull-replication {GetDatabaseName()}";
await hubStore.Maintenance.ForDatabase(hubStore.Database).SendAsync(new PutPullReplicationAsHubOperation(new PullReplicationDefinition(name)
{
MentorNode = "A"
}));
var pullReplication = new PullReplicationAsSink(hubDB, $"ConnectionString-{hubDB}", name)
{
MentorNode = "B", // this is the node were the data will be replicated to.
};
await AddWatcherToReplicationTopology((DocumentStore)minionStore, pullReplication, new[] { hub.WebUrl });
using (var dstSession = minionStore.OpenSession())
{
Assert.True(await WaitForDocumentInClusterAsync <User>(
minionNodes,
minionDB,
"users/1",
u => u.Name.Equals("Karmel"),
TimeSpan.FromSeconds(30)));
}
var minionUrl = minion.ServerStore.GetClusterTopology().GetUrlFromTag("B");
var minionServer = Servers.Single(s => s.WebUrl == minionUrl);
var handler = await InstantiateOutgoingTaskHandler(minionDB, minionServer);
Assert.True(WaitForValue(
() => ((OngoingTaskPullReplicationAsSink)handler.GetOngoingTasksInternal().OngoingTasksList.Single(t => t is OngoingTaskPullReplicationAsSink)).DestinationUrl != null,
true));
var mentorUrl = hub.ServerStore.GetClusterTopology().GetUrlFromTag("A");
var mentor = Servers.Single(s => s.WebUrl == mentorUrl);
var mentorDatabase = await mentor.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(hubDB);
var connections = await WaitForValueAsync(() => mentorDatabase.ReplicationLoader.OutgoingConnections.Count(), 3);
Assert.Equal(3, connections);
minionServer.CpuCreditsBalance.BackgroundTasksAlertRaised.Raise();
Assert.Equal(1,
await WaitForValueAsync(async() => (await minionStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(minionDB))).Topology.Rehabs.Count,
1));
await EnsureReplicatingAsync((DocumentStore)hubStore, (DocumentStore)minionStore);
connections = await WaitForValueAsync(() => mentorDatabase.ReplicationLoader.OutgoingConnections.Count(), 3);
Assert.Equal(3, connections);
}
}
public async Task FailoverOnHubNodeFail()
{
var clusterSize = 3;
var(_, hub) = await CreateRaftCluster(clusterSize);
var(minionNodes, minion) = await CreateRaftCluster(clusterSize);
var hubDB = GetDatabaseName();
var minionDB = GetDatabaseName();
var dstTopology = await CreateDatabaseInCluster(minionDB, clusterSize, minion.WebUrl);
var srcTopology = await CreateDatabaseInCluster(hubDB, clusterSize, hub.WebUrl);
using (var hubStore = new DocumentStore
{
Urls = new[] { hub.WebUrl },
Database = hubDB
}.Initialize())
using (var minionStore = new DocumentStore
{
Urls = new[] { minion.WebUrl },
Database = minionDB
}.Initialize())
{
using (var session = hubStore.OpenSession())
{
session.Advanced.WaitForReplicationAfterSaveChanges(timeout: TimeSpan.FromSeconds(10), replicas: clusterSize - 1);
session.Store(new User
{
Name = "Karmel"
}, "users/1");
session.SaveChanges();
}
var name = $"pull-replication {GetDatabaseName()}";
await hubStore.Maintenance.ForDatabase(hubStore.Database).SendAsync(new PutPullReplicationAsHubOperation(name));
// add pull replication with invalid discovery url to test the failover on database topology discovery
var pullReplication = new PullReplicationAsSink(hubDB, $"ConnectionString-{hubDB}", name)
{
MentorNode = "B", // this is the node were the data will be replicated to.
};
var urls = new List <string>();
foreach (var ravenServer in srcTopology.Servers)
{
urls.Add(ravenServer.WebUrl);
}
await AddWatcherToReplicationTopology((DocumentStore)minionStore, pullReplication, urls.ToArray());
using (var dstSession = minionStore.OpenSession())
{
Assert.True(await WaitForDocumentInClusterAsync <User>(
minionNodes,
minionDB,
"users/1",
u => u.Name.Equals("Karmel"),
TimeSpan.FromSeconds(30)));
}
var minionUrl = minion.ServerStore.GetClusterTopology().GetUrlFromTag("B");
var server = Servers.Single(s => s.WebUrl == minionUrl);
var handler = await InstantiateOutgoingTaskHandler(minionDB, server);
Assert.True(WaitForValue(
() => ((OngoingTaskPullReplicationAsSink)handler.GetOngoingTasksInternal().OngoingTasksList.Single(t => t is OngoingTaskPullReplicationAsSink)).DestinationUrl !=
null,
true));
var watcherTaskUrl = ((OngoingTaskPullReplicationAsSink)handler.GetOngoingTasksInternal().OngoingTasksList.Single(t => t is OngoingTaskPullReplicationAsSink)).DestinationUrl;
// dispose the hub node, from which we are currently pulling
DisposeServerAndWaitForFinishOfDisposal(Servers.Single(s => s.WebUrl == watcherTaskUrl));
using (var session = hubStore.OpenSession())
{
session.Advanced.WaitForReplicationAfterSaveChanges(timeout: TimeSpan.FromSeconds(10), replicas: clusterSize - 2);
session.Store(new User
{
Name = "Karmel2"
}, "users/2");
session.SaveChanges();
}
WaitForUserToContinueTheTest(minionStore);
using (var dstSession = minionStore.OpenSession())
{
Assert.True(await WaitForDocumentInClusterAsync <User>(
minionNodes,
minionDB,
"users/2",
u => u.Name.Equals("Karmel2"),
TimeSpan.FromSeconds(30)));
}
}
}
