public void CanFailoverReplicationBetweenTwoMultiTenantDatabases()
{
using (var store = new DocumentStore
{
DefaultDatabase = "FailoverTest",
Url = store1.Url,
Conventions =
{
FailoverBehavior = FailoverBehavior.AllowReadsFromSecondariesAndWritesToSecondaries
}
})
{
store.Initialize();
var replicationInformerForDatabase = store.GetReplicationInformerForDatabase(null);
replicationInformerForDatabase.UpdateReplicationInformationIfNeeded((ServerClient) store.DatabaseCommands)
.Wait();
using (var session = store.OpenSession())
{
session.Store(new Item());
session.SaveChanges();
}
WaitForDocument(store2.DatabaseCommands.ForDatabase("FailoverTest"), "items/1");
servers[0].Dispose();
using (var session = store.OpenSession())
{
var load = session.Load<Item>("items/1");
Assert.NotNull(load);
}
}
}
public void CanReplicateBetweenTwoMultiTenantDatabases()
{
var store1 = CreateStore();
var store2 = CreateStore();
store1.DatabaseCommands.EnsureDatabaseExists("FailoverTest");
store2.DatabaseCommands.EnsureDatabaseExists("FailoverTest");
SetupReplication(store1.DatabaseCommands.ForDatabase("FailoverTest"),
store2.Url + "/databases/FailoverTest");
using (var store = new DocumentStore
{
DefaultDatabase = "FailoverTest",
Url = store1.Url,
Conventions =
{
FailoverBehavior = FailoverBehavior.AllowReadsFromSecondariesAndWritesToSecondaries
}
})
{
store.Initialize();
var replicationInformerForDatabase = store.GetReplicationInformerForDatabase(null);
replicationInformerForDatabase.UpdateReplicationInformationIfNeeded((ServerClient) store.DatabaseCommands)
.Wait();
using (var session = store.OpenSession())
{
session.Store(new Item {});
session.SaveChanges();
}
WaitForDocument(store2.DatabaseCommands.ForDatabase("FailoverTest"), "items/1");
}
}
public void CanReplicateBetweenTwoMultiTenantDatabases()
{
using (var store = new DocumentStore
{
DefaultDatabase = "FailoverTest",
Url = store1.Url,
Conventions =
{
FailoverBehavior = FailoverBehavior.AllowReadsFromSecondariesAndWritesToSecondaries
}
})
{
store.Initialize();
var replicationInformerForDatabase = store.GetReplicationInformerForDatabase(null);
var databaseCommands = (ServerClient) store.DatabaseCommands;
replicationInformerForDatabase.UpdateReplicationInformationIfNeeded(databaseCommands)
.Wait();
var replicationDestinations = replicationInformerForDatabase.ReplicationDestinationsUrls;
Assert.NotEmpty(replicationDestinations);
using (var session = store.OpenSession())
{
session.Store(new Item());
session.SaveChanges();
}
var sanityCheck = store.DatabaseCommands.Head("items/1");
Assert.NotNull(sanityCheck);
WaitForDocument(store2.DatabaseCommands.ForDatabase("FailoverTest"), "items/1");
}
}
public void When_replicating_can_do_read_striping()
{
var store1 = CreateStore();
var store2 = CreateStore();
var store3 = CreateStore();
using (var session = store1.OpenSession())
{
session.Store(new Company());
session.SaveChanges();
}
SetupReplication(store1.DatabaseCommands, store2, store3);
WaitForDocument(store2.DatabaseCommands, "companies/1");
WaitForDocument(store3.DatabaseCommands, "companies/1");
PauseReplicationAsync(0, store1.DefaultDatabase).Wait();
PauseReplicationAsync(1, store2.DefaultDatabase).Wait();
PauseReplicationAsync(2, store3.DefaultDatabase).Wait();
using(var store = new DocumentStore
{
Url = store1.Url,
Conventions =
{
FailoverBehavior = FailoverBehavior.ReadFromAllServers
},
DefaultDatabase = store1.DefaultDatabase
})
{
store.Initialize();
var replicationInformerForDatabase = store.GetReplicationInformerForDatabase();
replicationInformerForDatabase.UpdateReplicationInformationIfNeeded((AsyncServerClient)store.AsyncDatabaseCommands)
.Wait();
Assert.Equal(2, replicationInformerForDatabase.ReplicationDestinationsUrls.Count);
foreach (var ravenDbServer in servers)
{
ravenDbServer.Server.ResetNumberOfRequests();
}
for (int i = 0; i < 6; i++)
{
using(var session = store.OpenSession())
{
Assert.NotNull(session.Load<Company>("companies/1"));
}
}
}
foreach (var ravenDbServer in servers)
{
Assert.Equal(2, ravenDbServer.Server.NumberOfRequests);
}
}
public async Task When_replicating_can_do_read_striping()
{
var store1 = CreateStore();
var store2 = CreateStore();
var store3 = CreateStore();
using (var session = store1.OpenAsyncSession())
{
await session.StoreAsync(new Company());
await session.SaveChangesAsync();
}
SetupReplication(store1.DatabaseCommands, store2, store3);
WaitForDocument(store2.DatabaseCommands, "companies/1");
WaitForDocument(store3.DatabaseCommands, "companies/1");
await PauseReplicationAsync(0, store1.DefaultDatabase);
await PauseReplicationAsync(1, store2.DefaultDatabase);
await PauseReplicationAsync(2, store3.DefaultDatabase);
using(var store = new DocumentStore
{
Url = store1.Url,
Conventions =
{
FailoverBehavior = FailoverBehavior.ReadFromAllServers
},
DefaultDatabase = store1.DefaultDatabase
})
{
store.Initialize();
var replicationInformerForDatabase = store.GetReplicationInformerForDatabase();
await replicationInformerForDatabase.UpdateReplicationInformationIfNeeded((AsyncServerClient)store.AsyncDatabaseCommands);
Assert.Equal(2, replicationInformerForDatabase.ReplicationDestinationsUrls.Count);
foreach (var ravenDbServer in servers)
{
ravenDbServer.Server.ResetNumberOfRequests();
Assert.Equal(0, ravenDbServer.Server.NumberOfRequests);
}
for (int i = 0; i < 6; i++)
{
using(var session = store.OpenAsyncSession())
{
Assert.NotNull(await session.LoadAsync<Company>("companies/1"));
}
}
}
foreach (var ravenDbServer in servers)
{
Assert.True(2 == ravenDbServer.Server.NumberOfRequests, string.Format("Server at port: {0}. Requests: #{1}", ravenDbServer.SystemDatabase.Configuration.Port, ravenDbServer.Server.NumberOfRequests));
}
}
public static void Main(string[] args)
{
var store1 = new DocumentStore
{
Url = Url1,
Conventions =
{
FailoverBehavior = FailoverBehavior.ReadFromAllServers
}
};
var store2 = new DocumentStore
{
Url = Url2
};
store1.Initialize();
store2.Initialize();
CreateDatabase(store1, DatabaseName);
Console.WriteLine("Created {0} database on {1}.", DatabaseName, store1.Url);
CreateDatabase(store2, DatabaseName);
Console.WriteLine("Created {0} database on {1}.", DatabaseName, store2.Url);
CreateReplication(store1, DatabaseName, Url2, DatabaseName);
Console.WriteLine("Created Replication document on {0}.", store1.Url);
var replicationInformerForDatabase = store1.GetReplicationInformerForDatabase(DatabaseName);
replicationInformerForDatabase.RefreshReplicationInformation((ServerClient)store1.DatabaseCommands.ForDatabase(DatabaseName));
using (var session = store1.OpenSession(DatabaseName))
{
session.Store(new User { Name = "Ayende" }, "users/ayende");
session.SaveChanges();
}
WaitForDocument(store2.DatabaseCommands.ForDatabase(DatabaseName), "users/ayende");
Console.WriteLine("Document users/ayende successfully replicated to {0}, ready for {1} server failure.", store2.Url, store1.Url);
Console.WriteLine("Press any key to continue...");
Console.ReadLine();
for (int i = 0; i < 12; i++)
{
using (var session = store1.OpenSession(DatabaseName))
{
session.Load<User>("users/ayende");
}
}
store1.Dispose();
store2.Dispose();
}
public async Task CanFailoverReplicationBetweenTwoMultiTenantDatabases()
{
var store1 = CreateStore();
var store2 = CreateStore();
store1.DatabaseCommands.GlobalAdmin.EnsureDatabaseExists("FailoverTest");
store2.DatabaseCommands.GlobalAdmin.EnsureDatabaseExists("FailoverTest");
SetupReplication(store1.DatabaseCommands.ForDatabase("FailoverTest"),
store2.Url + "/databases/FailoverTest");
using (var store = new DocumentStore
{
DefaultDatabase = "FailoverTest",
Url = store1.Url,
Conventions =
{
FailoverBehavior = FailoverBehavior.AllowReadsFromSecondariesAndWritesToSecondaries
}
})
{
store.Initialize();
var replicationInformerForDatabase = store.GetReplicationInformerForDatabase(null);
await replicationInformerForDatabase.UpdateReplicationInformationIfNeeded((ServerClient) store.DatabaseCommands);
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new Item {});
await session.SaveChangesAsync();
}
WaitForDocument(store2.DatabaseCommands.ForDatabase("FailoverTest"), "items/1");
servers[0].Dispose();
using (var session = store.OpenAsyncSession())
{
var load = await session.LoadAsync<Item>("items/1");
Assert.NotNull(load);
}
}
}
public void Can_avoid_read_striping()
{
var store1 = CreateStore();
var store2 = CreateStore();
var store3 = CreateStore();
using (var session = store1.OpenAsyncSession())
{
session.Store(new Company());
session.SaveChangesAsync().Wait();
}
SetupReplication(store1.DatabaseCommands, store2.Url, store3.Url);
WaitForDocument(store2.DatabaseCommands, "companies/1");
WaitForDocument(store3.DatabaseCommands, "companies/1");
using (var store = new DocumentStore
{
Url = store1.Url,
Conventions =
{
FailoverBehavior = FailoverBehavior.ReadFromAllServers
}
})
{
store.Initialize();
var replicationInformerForDatabase = store.GetReplicationInformerForDatabase();
replicationInformerForDatabase.UpdateReplicationInformationIfNeeded((ServerClient)store.DatabaseCommands)
.Wait();
Assert.Equal(2, replicationInformerForDatabase.ReplicationDestinations.Count);
foreach (var ravenDbServer in servers)
{
ravenDbServer.Server.ResetNumberOfRequests();
}
for (int i = 0; i < 6; i++)
{
using (var session = store.OpenAsyncSession(new OpenSessionOptions
{
ForceReadFromMaster = true
}))
{
Assert.NotNull(session.LoadAsync<Company>("companies/1").Result);
}
}
}
Assert.Equal(6, servers[0].Server.NumberOfRequests);
Assert.Equal(0, servers[1].Server.NumberOfRequests);
Assert.Equal(0, servers[2].Server.NumberOfRequests);
}
public async Task ClientShouldGetInformationFromSecondaryServerThatItsPrimaryServerMightBeUp()
{
var db1Url = store1.Url + "/databases/Northwind";
var db2Url = store2.Url + "/databases/Northwind";
SetupReplication(store1.DatabaseCommands, db2Url);
using (var store = new DocumentStore
{
DefaultDatabase = "Northwind",
Url = store1.Url,
Conventions =
{
FailoverBehavior = FailoverBehavior.AllowReadsFromSecondariesAndWritesToSecondaries
}
})
{
store.Initialize();
var replicationInformerForDatabase = store.GetReplicationInformerForDatabase("Northwind");
replicationInformerForDatabase.DelayTimeInMiliSec = 0;
await replicationInformerForDatabase.UpdateReplicationInformationIfNeeded((ServerClient) store.DatabaseCommands);
Assert.NotEmpty(replicationInformerForDatabase.ReplicationDestinations);
using (var session = store.OpenSession())
{
session.Store(new Item());
session.SaveChanges();
}
WaitForDocument(store2.DatabaseCommands, "items/1");
Assert.Equal(0, replicationInformerForDatabase.GetFailureCount(db1Url));
StopServer(server1);
// Fail few times so we will be sure that client does not try its primary url
for (int i = 0; i < 2; i++)
{
Assert.NotNull(store.DatabaseCommands.Get("items/1"));
}
Assert.True(replicationInformerForDatabase.GetFailureCount(db1Url) > 0);
var replicationTask = (await server2.Server.GetDatabaseInternal("Northwind")).StartupTasks.OfType<ReplicationTask>().First();
replicationTask.Heartbeats.Clear();
server1 = CreateServer(8001, server1DataDir);
Assert.NotNull(store1.DatabaseCommands.Get("items/1"));
while (true)
{
var name = db1Url.Replace("localhost", Environment.MachineName.ToLowerInvariant())
.Replace(".fiddler", "");
DateTime time;
if (replicationTask.Heartbeats.TryGetValue(name, out time))
{
break;
}
Thread.Sleep(100);
}
Assert.True(replicationInformerForDatabase.GetFailureCount(db1Url) > 0);
Assert.NotNull(store.DatabaseCommands.Get("items/1"));
Assert.True(replicationInformerForDatabase.GetFailureCount(db1Url) > 0);
Assert.NotNull(store.DatabaseCommands.Get("items/1"));
Assert.Equal(0, replicationInformerForDatabase.GetFailureCount(db1Url));
}
}
public void ClientShouldGetInformationFromSecondaryServerThatItsPrimaryServerMightBeUp()
{
server1 = CreateServer(8111, "D1");
server2 = CreateServer(8112, "D2");
store1 = new DocumentStore
{
DefaultDatabase = "Northwind1",
Url = server1.Database.ServerUrl
};
store1.Initialize();
store2 = new DocumentStore
{
DefaultDatabase = "Northwind2",
Url = server2.Database.ServerUrl
};
store2.Initialize();
store1.DatabaseCommands.CreateDatabase(
new DatabaseDocument
{
Id = "Northwind1",
Settings = {{"Raven/ActiveBundles", "replication"}, {"Raven/DataDir", @"~\D1\N"}}
});
store2.DatabaseCommands.CreateDatabase(
new DatabaseDocument
{
Id = "Northwind2",
Settings = {{"Raven/ActiveBundles", "replication"}, {"Raven/DataDir", @"~\D2\N"}}
});
var db1Url = store1.Url + "/databases/Northwind1";
var db2Url = store2.Url + "/databases/Northwind2";
SetupReplication(store1.DatabaseCommands, db2Url);
using (var store = new DocumentStore
{
DefaultDatabase = "Northwind1",
Url = store1.Url,
Conventions =
{
FailoverBehavior = FailoverBehavior.AllowReadsFromSecondariesAndWritesToSecondaries
}
})
{
store.Initialize();
var replicationInformerForDatabase = store.GetReplicationInformerForDatabase("Northwind1");
replicationInformerForDatabase
.UpdateReplicationInformationIfNeeded((ServerClient) store.DatabaseCommands)
.Wait();
Assert.NotEmpty(replicationInformerForDatabase.ReplicationDestinations);
using (var session = store.OpenSession())
{
session.Store(new Item());
session.SaveChanges();
}
WaitForDocument(store2.DatabaseCommands, "items/1");
Assert.Equal(0, replicationInformerForDatabase.GetFailureCount(db1Url));
StopServer(server1);
// Fail few times so we will be sure that client does not try its primary url
for (int i = 0; i < 2; i++)
{
Assert.NotNull(store.DatabaseCommands.Get("items/1"));
}
Assert.True(replicationInformerForDatabase.GetFailureCount(db1Url) > 0);
var replicationTask =
server2.Server.GetDatabaseInternal("Northwind2").Result.StartupTasks.OfType<ReplicationTask>().First();
replicationTask.Heartbeats.Clear();
server1 = StartServer(server1);
Assert.NotNull(store1.DatabaseCommands.Get("items/1"));
while (true)
{
DateTime time;
if (replicationTask.Heartbeats.TryGetValue(db1Url.Replace("localhost", Environment.MachineName.ToLowerInvariant()), out time))
{
break;
}
Thread.Sleep(100);
}
Assert.True(replicationInformerForDatabase.GetFailureCount(db1Url) > 0);
Assert.NotNull(store.DatabaseCommands.Get("items/1"));
Assert.True(replicationInformerForDatabase.GetFailureCount(db1Url) > 0);
Assert.NotNull(store.DatabaseCommands.Get("items/1"));
Assert.Equal(0, replicationInformerForDatabase.GetFailureCount(db1Url));
}
}
public void CanFailoverReplicationBetweenTwoMultiTenantDatabases_WithExplicitUrl()
{
var store1 = CreateStore();
var store2 = CreateStore();
store1.DatabaseCommands.EnsureDatabaseExists("FailoverTest");
store2.DatabaseCommands.EnsureDatabaseExists("FailoverTest");
SetupReplication(store1.DatabaseCommands.ForDatabase("FailoverTest"),
store2.Url + "databases/FailoverTest");
using (var store = new DocumentStore
{
DefaultDatabase = "FailoverTest",
Url = store1.Url + "databases/FailoverTest",
Conventions =
{
FailoverBehavior = FailoverBehavior.AllowReadsFromSecondariesAndWritesToSecondaries
}
})
{
store.Initialize();
var replicationInformerForDatabase = store.GetReplicationInformerForDatabase("FailoverTest");
replicationInformerForDatabase.UpdateReplicationInformationIfNeeded((ServerClient) store.DatabaseCommands)
.Wait();
using (var session = store.OpenSession())
{
session.Store(new Item {});
session.SaveChanges();
}
WaitForDocument(store2.DatabaseCommands.ForDatabase("FailoverTest"), "items/1");
servers[0].Dispose();
while (true)
{
try
{
using (var session = store.OpenSession())
{
var load = session.Load<Item>("items/1");
Assert.NotNull(load);
}
return;
}
catch (Exception)
{
Debugger.Launch();
}
}
}
}
