protected static async Task <OngoingTasksHandler> InstantiateOutgoingTaskHandler(string name, RavenServer server)
{
Assert.True(server.ServerStore.DatabasesLandlord.DatabasesCache.TryGetValue(name, out var db));
var database = await db;
var handler = new OngoingTasksHandler();
var ctx = new RequestHandlerContext
{
RavenServer = server,
Database = database,
HttpContext = new DefaultHttpContext()
};
handler.Init(ctx);
return(handler);
}
public async Task ExternalReplicationFailover()
{
var clusterSize = 3;
var srcLeader = await CreateRaftClusterAndGetLeader(clusterSize);
var dstLeader = await CreateRaftClusterAndGetLeader(clusterSize);
var dstDB = GetDatabaseName();
var srcDB = GetDatabaseName();
var dstTopology = await CreateDatabaseInCluster(dstDB, clusterSize, dstLeader.WebUrl);
var srcTopology = await CreateDatabaseInCluster(srcDB, clusterSize, srcLeader.WebUrl);
var srcStore = new DocumentStore()
{
Urls = new[] { srcLeader.WebUrl },
Database = srcDB,
}.Initialize();
using (var session = srcStore.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/1");
session.SaveChanges();
Assert.True(await WaitForDocumentInClusterAsync <User>(
session as DocumentSession,
"users/1",
u => u.Name.Equals("Karmel"),
TimeSpan.FromSeconds(clusterSize + 5)));
}
// add watcher with invalid url to test the failover on database topology discovery
var watcher = new ExternalReplication(dstDB, new[] { "http://127.0.0.1:1234", dstLeader.WebUrl });
var res = await AddWatcherToReplicationTopology((DocumentStore)srcStore, watcher);
var dstStore = new DocumentStore
{
Urls = new[] { dstLeader.WebUrl },
Database = watcher.Database,
}.Initialize();
var dstSession = dstStore.OpenSession();
dstSession.Load <User>("Karmel");
Assert.True(await WaitForDocumentInClusterAsync <User>(
dstSession as DocumentSession,
"users/1",
u => u.Name.Equals("Karmel"),
TimeSpan.FromSeconds(60)));
var responsibale = srcLeader.ServerStore.GetClusterTopology().GetUrlFromTag(res.ResponsibleNode);
var server = Servers.Single(s => s.WebUrl == responsibale);
Assert.True(WaitForValue(() => OngoingTasksHandler.GetOngoingTasksFor(srcDB, server.ServerStore).OngoingTasksList.Single(t => t is OngoingTaskReplication).As <OngoingTaskReplication>().DestinationUrl != null, true));
var watcherTaskUrl = OngoingTasksHandler.GetOngoingTasksFor(srcDB, server.ServerStore).OngoingTasksList.Single(t => t is OngoingTaskReplication).As <OngoingTaskReplication>().DestinationUrl;
// fail the node to to where the data is sent
DisposeServerAndWaitForFinishOfDisposal(Servers.Single(s => s.WebUrl == watcherTaskUrl));
using (var session = srcStore.OpenSession())
{
session.Store(new User {
Name = "Karmel2"
}, "users/2");
session.SaveChanges();
Assert.True(await WaitForDocumentInClusterAsync <User>(
session as DocumentSession,
"users/2",
u => u.Name.Equals("Karmel2"),
TimeSpan.FromSeconds(clusterSize + 5)));
}
WaitForUserToContinueTheTest(dstStore as DocumentStore);
Assert.True(WaitForDocument(dstStore, "users/2", 30_000));
srcStore.Dispose();
dstSession.Dispose();
dstStore.Dispose();
}
public async Task CanAddAndModifySingleWatcher()
{
var clusterSize = 3;
var databaseName = "ReplicationTestDB";
var leader = await CreateRaftClusterAndGetLeader(clusterSize);
ExternalReplication watcher;
using (var store = new DocumentStore()
{
Urls = new[] { leader.WebUrl },
Database = databaseName,
Conventions =
{
DisableTopologyUpdates = true
}
}.Initialize())
{
var doc = new DatabaseRecord(databaseName);
var databaseResult = await store.Admin.Server.SendAsync(new CreateDatabaseOperation(doc, clusterSize));
Assert.Equal(clusterSize, databaseResult.Topology.AllNodes.Count());
foreach (var server in Servers)
{
await server.ServerStore.Cluster.WaitForIndexNotification(databaseResult.RaftCommandIndex);
}
foreach (var server in Servers)
{
await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName);
}
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "Karmel" }, "users/1");
await session.SaveChangesAsync();
}
Assert.True(await WaitForDocumentInClusterAsync <User>(
databaseResult.Topology,
databaseName,
"users/1",
u => u.Name.Equals("Karmel"),
TimeSpan.FromSeconds(clusterSize + 5)));
doc = new DatabaseRecord("Watcher");
var res = await store.Admin.Server.SendAsync(new CreateDatabaseOperation(doc));
var node = Servers.Single(x => x.WebUrl == res.NodesAddedTo[0]);
await node.ServerStore.Cluster.WaitForIndexNotification(res.RaftCommandIndex);
await node.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore("Watcher");
watcher = new ExternalReplication
{
Database = "Watcher",
Url = res.NodesAddedTo[0],
Name = "MyExternalReplication1"
};
await AddWatcherToReplicationTopology((DocumentStore)store, watcher);
}
var tasks = OngoingTasksHandler.GetOngoingTasksFor(databaseName, leader.ServerStore);
Assert.Equal(1, tasks.OngoingTasksList.Count);
var repTask = tasks.OngoingTasksList[0] as OngoingTaskReplication;
Assert.Equal(repTask?.DestinationDatabase, watcher.Database);
Assert.Equal(repTask?.DestinationUrl, watcher.Url);
Assert.Equal(repTask?.TaskName, watcher.Name);
watcher.TaskId = Convert.ToInt64(repTask?.TaskId);
using (var store = new DocumentStore
{
Urls = new[] { watcher.Url },
Database = watcher.Database,
Conventions =
{
DisableTopologyUpdates = true
}
}.Initialize())
{
Assert.True(WaitForDocument <User>(store, "users/1", u => u.Name == "Karmel"));
}
using (var store = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName,
Conventions =
{
DisableTopologyUpdates = true
}
}.Initialize())
{
var doc = new DatabaseRecord("Watcher2");
var res = await store.Admin.Server.SendAsync(new CreateDatabaseOperation(doc));
var node = Servers.Single(x => x.WebUrl == res.NodesAddedTo[0]);
await node.ServerStore.Cluster.WaitForIndexNotification(res.RaftCommandIndex);
await node.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore("Watcher2");
//modify watcher
watcher.Database = "Watcher2";
watcher.Url = res.NodesAddedTo[0];
watcher.Name = "MyExternalReplication2";
await AddWatcherToReplicationTopology((DocumentStore)store, watcher);
}
tasks = OngoingTasksHandler.GetOngoingTasksFor(databaseName, leader.ServerStore);
Assert.Equal(1, tasks.OngoingTasksList.Count);
repTask = tasks.OngoingTasksList[0] as OngoingTaskReplication;
Assert.Equal(repTask?.DestinationDatabase, watcher.Database);
Assert.Equal(repTask?.DestinationUrl, watcher.Url);
Assert.Equal(repTask?.TaskName, watcher.Name);
using (var store = new DocumentStore
{
Urls = new[] { watcher.Url },
Database = watcher.Database,
Conventions =
{
DisableTopologyUpdates = true
}
}.Initialize())
{
Assert.True(WaitForDocument <User>(store, "users/1", u => u.Name == "Karmel"));
}
//delete watcher
using (var store = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName,
Conventions =
{
DisableTopologyUpdates = true
}
}.Initialize())
{
await DeleteOngoingTask((DocumentStore)store, watcher.TaskId, OngoingTaskType.Replication);
tasks = OngoingTasksHandler.GetOngoingTasksFor(databaseName, leader.ServerStore);
Assert.Equal(0, tasks.OngoingTasksList.Count);
}
}
public async Task EnsureReplicationToWatchers(bool useSsl)
{
var clusterSize = 3;
var databaseName = GetDatabaseName();
var leader = await CreateRaftClusterAndGetLeader(clusterSize, useSsl : useSsl);
var watchers = new List <ExternalReplication>();
X509Certificate2 adminCertificate = null;
if (useSsl)
{
adminCertificate = AskServerForClientCertificate(_selfSignedCertFileName, new Dictionary <string, DatabaseAccess>(), SecurityClearance.ClusterAdmin, server: leader);
}
using (var store = new DocumentStore()
{
Urls = new[] { leader.WebUrl },
Database = databaseName,
Certificate = adminCertificate,
Conventions =
{
DisableTopologyUpdates = true
}
}.Initialize())
{
var doc = new DatabaseRecord(databaseName);
var databaseResult = await store.Admin.Server.SendAsync(new CreateDatabaseOperation(doc, clusterSize));
Assert.Equal(clusterSize, databaseResult.Topology.AllNodes.Count());
foreach (var server in Servers)
{
await server.ServerStore.Cluster.WaitForIndexNotification(databaseResult.RaftCommandIndex);
}
foreach (var server in Servers)
{
await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName);
}
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "Karmel" }, "users/1");
await session.SaveChangesAsync();
}
Assert.True(await WaitForDocumentInClusterAsync <User>(
databaseResult.Topology,
databaseName,
"users/1",
u => u.Name.Equals("Karmel"),
TimeSpan.FromSeconds(clusterSize + 5),
adminCertificate));
for (var i = 0; i < 5; i++)
{
doc = new DatabaseRecord($"Watcher{i}");
var res = await store.Admin.Server.SendAsync(new CreateDatabaseOperation(doc));
var server = Servers.Single(x => x.WebUrl == res.NodesAddedTo[0]);
await server.ServerStore.Cluster.WaitForIndexNotification(res.RaftCommandIndex);
await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore($"Watcher{i}");
var watcher = new ExternalReplication($"Watcher{i}", res.NodesAddedTo.ToArray());
watchers.Add(watcher);
await AddWatcherToReplicationTopology((DocumentStore)store, watcher);
}
}
foreach (var watcher in watchers)
{
using (var store = new DocumentStore
{
Urls = watcher.TopologyDiscoveryUrls,
Database = watcher.Database,
Certificate = adminCertificate,
Conventions =
{
DisableTopologyUpdates = true
}
}.Initialize())
{
Assert.True(WaitForDocument <User>(store, "users/1", u => u.Name == "Karmel"));
}
}
var count = 0;
foreach (var server in Servers)
{
count += OngoingTasksHandler.GetOngoingTasksFor(databaseName, server.ServerStore).OngoingTasksList.Count(t => t is OngoingTaskReplication rep && rep.Status != OngoingTaskReplication.ReplicationStatus.NotOnThisNode);
}
Assert.Equal(5, count);
}