public void CanGetConnectionStringByName()
{
using (var store = GetDocumentStore())
{
var ravenConnectionStrings = new List <RavenConnectionString>();
var sqlConnectionStrings = new List <SqlConnectionString>();
var ravenConnectionStr = new RavenConnectionString()
{
Name = "RavenConnectionString",
TopologyDiscoveryUrls = new[] { "http://127.0.0.1:8080" },
Database = "Northwind",
};
var sqlConnectionStr = new SqlConnectionString
{
Name = "SqlConnectionString",
ConnectionString = SqlEtlTests.GetConnectionString(store)
};
ravenConnectionStrings.Add(ravenConnectionStr);
sqlConnectionStrings.Add(sqlConnectionStr);
store.Maintenance.Server.Send(new PutConnectionStringOperation <RavenConnectionString>(ravenConnectionStr, store.Database));
store.Maintenance.Server.Send(new PutConnectionStringOperation <SqlConnectionString>(sqlConnectionStr, store.Database));
var result = store.Maintenance.Server.Send(new GetConnectionStringsOperation(store.Database, connectionStringName: sqlConnectionStr.Name, type: sqlConnectionStr.Type));
Assert.True(result.SqlConnectionStrings.Count > 0);
Assert.True(result.RavenConnectionStrings.Count == 0);
}
}
internal static async Task <AddEtlOperationResult> AddEtl(IDocumentStore source, string destination, string[] urls, string mentor)
{
var connectionStringName = $"RavenEtl_From{source.Database}_To{destination}";
var config = new RavenEtlConfiguration()
{
Name = connectionStringName,
ConnectionStringName = connectionStringName,
LoadRequestTimeoutInSec = 10,
MentorNode = mentor,
Transforms = new List <Transformation>
{
new Transformation
{
Name = $"ETL : {connectionStringName}",
ApplyToAllDocuments = true,
IsEmptyScript = true
}
}
};
var connectionString = new RavenConnectionString
{
Name = connectionStringName,
Database = destination,
TopologyDiscoveryUrls = urls,
};
var result = await source.Maintenance.SendAsync(new PutConnectionStringOperation <RavenConnectionString>(connectionString));
Assert.NotNull(result.RaftCommandIndex);
return(await source.Maintenance.SendAsync(new AddEtlOperation <RavenConnectionString>(config)));
}
public void CanReachDatabaseAdminEndpointWithDatabaseAdminPermission()
{
var serverCertPath = SetupServerAuthentication();
var dbName = GetDatabaseName();
var adminCert = AskServerForClientCertificate(serverCertPath, new Dictionary <string, DatabaseAccess>(), SecurityClearance.ClusterAdmin);
var userCert = AskServerForClientCertificate(serverCertPath, new Dictionary <string, DatabaseAccess>
{
[dbName] = DatabaseAccess.Admin
});
using (var store = GetDocumentStore(new Options
{
AdminCertificate = adminCert,
ClientCertificate = userCert,
ModifyDatabaseName = s => dbName
}))
{
var ravenConnectionStr = new RavenConnectionString()
{
Name = $"RavenConnectionString",
TopologyDiscoveryUrls = new [] { $"http://127.0.0.1:8080" },
Database = dbName,
};
store.Maintenance.Send(new PutConnectionStringOperation <RavenConnectionString>(ravenConnectionStr)); // DatabaseAdmin operation
var result = store.Maintenance.Send(new GetConnectionStringsOperation(store.Database, ConnectionStringType.Raven));
Assert.NotNull(result.RavenConnectionStrings);
}
}
public void CannotReachDatabaseAdminEndpointWithoutDatabaseAdminPermission()
{
var serverCertPath = SetupServerAuthentication();
var dbName = GetDatabaseName();
var adminCert = AskServerForClientCertificate(serverCertPath, new Dictionary <string, DatabaseAccess>(), SecurityClearance.ClusterAdmin);
var userCert = AskServerForClientCertificate(serverCertPath, new Dictionary <string, DatabaseAccess>
{
[dbName] = DatabaseAccess.ReadWrite
});
using (var store = GetDocumentStore(new Options
{
AdminCertificate = adminCert,
ClientCertificate = userCert,
ModifyDatabaseName = s => dbName
}))
{
var ravenConnectionStr = new RavenConnectionString()
{
Name = $"RavenConnectionString",
TopologyDiscoveryUrls = new[] { $"http://127.0.0.1:8080" },
Database = dbName,
};
Assert.Throws <AuthorizationException>(() =>
{
store.Maintenance.Send(new PutConnectionStringOperation <RavenConnectionString>(ravenConnectionStr)); // DatabaseAdmin operation
});
}
}
public void CanGetConnectionStringByName()
{
using (var store = GetDocumentStore())
{
var ravenConnectionStrings = new List <RavenConnectionString>();
var sqlConnectionStrings = new List <SqlConnectionString>();
var ravenConnectionStr = new RavenConnectionString()
{
Name = "RavenConnectionString",
TopologyDiscoveryUrls = new[] { "http://127.0.0.1:8080" },
Database = "Northwind",
};
var sqlConnectionStr = new SqlConnectionString
{
Name = "SqlConnectionString",
ConnectionString = MssqlConnectionString.Instance.VerifiedConnectionString.Value + $";Initial Catalog={store.Database}"
};
ravenConnectionStrings.Add(ravenConnectionStr);
sqlConnectionStrings.Add(sqlConnectionStr);
var result1 = store.Maintenance.Send(new PutConnectionStringOperation <RavenConnectionString>(ravenConnectionStr));
Assert.NotNull(result1.RaftCommandIndex);
var result2 = store.Maintenance.Send(new PutConnectionStringOperation <SqlConnectionString>(sqlConnectionStr));
Assert.NotNull(result2.RaftCommandIndex);
var result = store.Maintenance.Send(new GetConnectionStringsOperation(connectionStringName: sqlConnectionStr.Name, type: sqlConnectionStr.Type));
Assert.True(result.SqlConnectionStrings.Count > 0);
Assert.True(result.RavenConnectionStrings.Count == 0);
}
}
public void CanGetAllConnectionStrings()
{
using (var store = GetDocumentStore())
{
var ravenConnectionStrings = new List <RavenConnectionString>();
var sqlConnectionStrings = new List <SqlConnectionString>();
var elasticSearchConnectionStrings = new List <ElasticSearchConnectionString>();
for (var i = 0; i < 5; i++)
{
var ravenConnectionStr = new RavenConnectionString()
{
Name = $"RavenConnectionString{i}",
TopologyDiscoveryUrls = new[] { $"http://127.0.0.1:808{i}" },
Database = "Northwind",
};
var sqlConnectionStr = new SqlConnectionString
{
Name = $"SqlConnectionString{i}",
ConnectionString = MssqlConnectionString.Instance.VerifiedConnectionString.Value + $";Initial Catalog={store.Database}"
};
var elasticConnectionStr = new ElasticSearchConnectionString
{
Name = $"ElasticConnectionString{i}",
Nodes = new[] { $"http://127.0.0.1:808{i}" },
};
ravenConnectionStrings.Add(ravenConnectionStr);
sqlConnectionStrings.Add(sqlConnectionStr);
elasticSearchConnectionStrings.Add(elasticConnectionStr);
var result1 = store.Maintenance.Send(new PutConnectionStringOperation <RavenConnectionString>(ravenConnectionStr));
Assert.NotNull(result1.RaftCommandIndex);
var result2 = store.Maintenance.Send(new PutConnectionStringOperation <SqlConnectionString>(sqlConnectionStr));
Assert.NotNull(result2.RaftCommandIndex);
var result3 = store.Maintenance.Send(new PutConnectionStringOperation <ElasticSearchConnectionString>(elasticConnectionStr));
Assert.NotNull(result3.RaftCommandIndex);
}
var result = store.Maintenance.Send(new GetConnectionStringsOperation());
Assert.NotNull(result.SqlConnectionStrings);
Assert.NotNull(result.RavenConnectionStrings);
Assert.NotNull(result.ElasticSearchConnectionStrings);
for (var i = 0; i < 5; i++)
{
result.SqlConnectionStrings.TryGetValue($"SqlConnectionString{i}", out var sql);
Assert.Equal(sql?.ConnectionString, sqlConnectionStrings[i].ConnectionString);
result.ElasticSearchConnectionStrings.TryGetValue($"ElasticConnectionString{i}", out var elastic);
Assert.Equal(elastic?.Nodes, elasticSearchConnectionStrings[i].Nodes);
result.RavenConnectionStrings.TryGetValue($"RavenConnectionString{i}", out var raven);
Assert.Equal(raven?.TopologyDiscoveryUrls, ravenConnectionStrings[i].TopologyDiscoveryUrls);
Assert.Equal(raven?.Database, ravenConnectionStrings[i].Database);
}
}
}
public async Task CanExcludeDatabase()
{
using (var store = GetDocumentStore())
{
var serverWideExternalReplication = new ServerWideExternalReplication
{
Disabled = true,
TopologyDiscoveryUrls = new[] { store.Urls.First() }
};
var result = await store.Maintenance.Server.SendAsync(new PutServerWideExternalReplicationOperation(serverWideExternalReplication));
serverWideExternalReplication.Name = result.Name;
var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
Assert.Equal(1, record.ExternalReplications.Count);
Assert.Equal(1, record.RavenConnectionStrings.Count);
var dbName = $"db/{Guid.NewGuid()}";
var csName = $"cs/{Guid.NewGuid()}";
var connectionString = new RavenConnectionString
{
Name = csName,
Database = dbName,
TopologyDiscoveryUrls = new[] { "http://127.0.0.1:12345" }
};
var putConnectionStringResult = await store.Maintenance.SendAsync(new PutConnectionStringOperation <RavenConnectionString>(connectionString));
Assert.NotNull(putConnectionStringResult.RaftCommandIndex);
var externalReplication = new ExternalReplication(dbName, csName)
{
Name = "Regular Task",
Disabled = true
};
await store.Maintenance.SendAsync(new UpdateExternalReplicationOperation(externalReplication));
record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
Assert.Equal(2, record.ExternalReplications.Count);
Assert.Equal(2, record.RavenConnectionStrings.Count);
serverWideExternalReplication.ExcludedDatabases = new[] { store.Database };
await store.Maintenance.Server.SendAsync(new PutServerWideExternalReplicationOperation(serverWideExternalReplication));
record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
Assert.Equal(1, record.ExternalReplications.Count);
Assert.Equal(1, record.RavenConnectionStrings.Count);
Assert.Equal(externalReplication.Name, record.ExternalReplications.First().Name);
}
}
public void CanAddAndRemoveConnectionStrings()
{
using (var store = GetDocumentStore())
{
var ravenConnectionString = new RavenConnectionString()
{
Name = "RavenConnectionString",
TopologyDiscoveryUrls = new[] { "http://localhost:8080" },
Database = "Northwind",
};
var result0 = store.Maintenance.Send(new PutConnectionStringOperation <RavenConnectionString>(ravenConnectionString));
Assert.NotNull(result0.RaftCommandIndex);
var sqlConnectionString = new SqlConnectionString
{
Name = "SqlConnectionString",
ConnectionString = MssqlConnectionString.Instance.VerifiedConnectionString.Value + $";Initial Catalog={store.Database}",
};
var result1 = store.Maintenance.Send(new PutConnectionStringOperation <SqlConnectionString>(sqlConnectionString));
Assert.NotNull(result1.RaftCommandIndex);
DatabaseRecord record;
using (Server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
record = Server.ServerStore.Cluster.ReadDatabase(context, store.Database);
}
Assert.True(record.RavenConnectionStrings.ContainsKey("RavenConnectionString"));
Assert.Equal(ravenConnectionString.Name, record.RavenConnectionStrings["RavenConnectionString"].Name);
Assert.Equal(ravenConnectionString.TopologyDiscoveryUrls, record.RavenConnectionStrings["RavenConnectionString"].TopologyDiscoveryUrls);
Assert.Equal(ravenConnectionString.Database, record.RavenConnectionStrings["RavenConnectionString"].Database);
Assert.True(record.SqlConnectionStrings.ContainsKey("SqlConnectionString"));
Assert.Equal(sqlConnectionString.Name, record.SqlConnectionStrings["SqlConnectionString"].Name);
Assert.Equal(sqlConnectionString.ConnectionString, record.SqlConnectionStrings["SqlConnectionString"].ConnectionString);
var result3 = store.Maintenance.Send(new RemoveConnectionStringOperation <RavenConnectionString>(ravenConnectionString));
Assert.NotNull(result3.RaftCommandIndex);
var result4 = store.Maintenance.Send(new RemoveConnectionStringOperation <SqlConnectionString>(sqlConnectionString));
Assert.NotNull(result4.RaftCommandIndex);
using (Server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
record = Server.ServerStore.Cluster.ReadDatabase(context, store.Database);
}
Assert.False(record.RavenConnectionStrings.ContainsKey("RavenConnectionString"));
Assert.False(record.SqlConnectionStrings.ContainsKey("SqlConnectionString"));
}
}
public void CanUpdateConnectionStrings()
{
using (var store = GetDocumentStore())
{
var ravenConnectionString = new RavenConnectionString()
{
Name = "RavenConnectionString",
TopologyDiscoveryUrls = new[] { "http://127.0.0.1:8080" },
Database = "Northwind",
};
var result1 = store.Maintenance.Send(new PutConnectionStringOperation <RavenConnectionString>(ravenConnectionString));
Assert.NotNull(result1.RaftCommandIndex);
var sqlConnectionString = new SqlConnectionString
{
Name = "SqlConnectionString",
ConnectionString = MssqlConnectionString.Instance.VerifiedConnectionString.Value + $";Initial Catalog={store.Database}",
};
var result2 = store.Maintenance.Send(new PutConnectionStringOperation <SqlConnectionString>(sqlConnectionString));
Assert.NotNull(result2.RaftCommandIndex);
//update url
ravenConnectionString.TopologyDiscoveryUrls = new[] { "http://127.0.0.1:8081" };
var result3 = store.Maintenance.Send(new PutConnectionStringOperation <RavenConnectionString>(ravenConnectionString));
Assert.NotNull(result3.RaftCommandIndex);
//update name : need to remove the old entry
var result4 = store.Maintenance.Send(new RemoveConnectionStringOperation <SqlConnectionString>(sqlConnectionString));
Assert.NotNull(result4.RaftCommandIndex);
sqlConnectionString.Name = "New-Name";
var result5 = store.Maintenance.Send(new PutConnectionStringOperation <SqlConnectionString>(sqlConnectionString));
Assert.NotNull(result5.RaftCommandIndex);
DatabaseRecord record;
using (Server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
record = Server.ServerStore.Cluster.ReadDatabase(context, store.Database);
}
Assert.True(record.RavenConnectionStrings.ContainsKey("RavenConnectionString"));
Assert.Equal("http://127.0.0.1:8081", record.RavenConnectionStrings["RavenConnectionString"].TopologyDiscoveryUrls.First());
Assert.False(record.SqlConnectionStrings.ContainsKey("SqlConnectionString"));
Assert.True(record.SqlConnectionStrings.ContainsKey("New-Name"));
Assert.Equal(sqlConnectionString.ConnectionString, record.SqlConnectionStrings["New-Name"].ConnectionString);
}
}
public void CanAddAndRemoveConnectionStrings()
{
using (var store = GetDocumentStore())
{
var ravenConnectionString = new RavenConnectionString()
{
Name = "RavenConnectionString",
TopologyDiscoveryUrls = new[] { "http://localhost:8080" },
Database = "Northwind",
};
store.Maintenance.Server.Send(new PutConnectionStringOperation <RavenConnectionString>(ravenConnectionString, store.Database));
var sqlConnectionString = new SqlConnectionString
{
Name = "SqlConnectionString",
ConnectionString = SqlEtlTests.GetConnectionString(store),
};
store.Maintenance.Server.Send(new PutConnectionStringOperation <SqlConnectionString>(sqlConnectionString, store.Database));
DatabaseRecord record;
using (Server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
record = Server.ServerStore.Cluster.ReadDatabase(context, store.Database);
}
Assert.True(record.RavenConnectionStrings.ContainsKey("RavenConnectionString"));
Assert.Equal(ravenConnectionString.Name, record.RavenConnectionStrings["RavenConnectionString"].Name);
Assert.Equal(ravenConnectionString.TopologyDiscoveryUrls, record.RavenConnectionStrings["RavenConnectionString"].TopologyDiscoveryUrls);
Assert.Equal(ravenConnectionString.Database, record.RavenConnectionStrings["RavenConnectionString"].Database);
Assert.True(record.SqlConnectionStrings.ContainsKey("SqlConnectionString"));
Assert.Equal(sqlConnectionString.Name, record.SqlConnectionStrings["SqlConnectionString"].Name);
Assert.Equal(sqlConnectionString.ConnectionString, record.SqlConnectionStrings["SqlConnectionString"].ConnectionString);
store.Maintenance.Server.Send(new RemoveConnectionStringOperation <RavenConnectionString>(ravenConnectionString, store.Database));
store.Maintenance.Server.Send(new RemoveConnectionStringOperation <SqlConnectionString>(sqlConnectionString, store.Database));
using (Server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
record = Server.ServerStore.Cluster.ReadDatabase(context, store.Database);
}
Assert.False(record.RavenConnectionStrings.ContainsKey("RavenConnectionString"));
Assert.False(record.SqlConnectionStrings.ContainsKey("SqlConnectionString"));
}
}
public void CanUpdateConnectionStrings()
{
using (var store = GetDocumentStore())
{
var ravenConnectionString = new RavenConnectionString()
{
Name = "RavenConnectionString",
TopologyDiscoveryUrls = new[] { "http://127.0.0.1:8080" },
Database = "Northwind",
};
store.Maintenance.Server.Send(new PutConnectionStringOperation <RavenConnectionString>(ravenConnectionString, store.Database));
var sqlConnectionString = new SqlConnectionString
{
Name = "SqlConnectionString",
ConnectionString = SqlEtlTests.GetConnectionString(store),
};
store.Maintenance.Server.Send(new PutConnectionStringOperation <SqlConnectionString>(sqlConnectionString, store.Database));
//update url
ravenConnectionString.TopologyDiscoveryUrls = new[] { "http://127.0.0.1:8081" };
store.Maintenance.Server.Send(new PutConnectionStringOperation <RavenConnectionString>(ravenConnectionString, store.Database));
//update name : need to remove the old entry
store.Maintenance.Server.Send(new RemoveConnectionStringOperation <SqlConnectionString>(sqlConnectionString, store.Database));
sqlConnectionString.Name = "New-Name";
store.Maintenance.Server.Send(new PutConnectionStringOperation <SqlConnectionString>(sqlConnectionString, store.Database));
DatabaseRecord record;
using (Server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
record = Server.ServerStore.Cluster.ReadDatabase(context, store.Database);
}
Assert.True(record.RavenConnectionStrings.ContainsKey("RavenConnectionString"));
Assert.Equal("http://127.0.0.1:8081", record.RavenConnectionStrings["RavenConnectionString"].TopologyDiscoveryUrls.First());
Assert.False(record.SqlConnectionStrings.ContainsKey("SqlConnectionString"));
Assert.True(record.SqlConnectionStrings.ContainsKey("New-Name"));
Assert.Equal(sqlConnectionString.ConnectionString, record.SqlConnectionStrings["New-Name"].ConnectionString);
}
}
public void CanGetAllConnectionStrings()
{
using (var store = GetDocumentStore())
{
var ravenConnectionStrings = new List <RavenConnectionString>();
var sqlConnectionStrings = new List <SqlConnectionString>();
for (var i = 0; i < 5; i++)
{
var ravenConnectionStr = new RavenConnectionString()
{
Name = $"RavenConnectionString{i}",
TopologyDiscoveryUrls = new[] { $"http://127.0.0.1:808{i}" },
Database = "Northwind",
};
var sqlConnectionStr = new SqlConnectionString
{
Name = $"SqlConnectionString{i}",
ConnectionString = SqlEtlTests.GetConnectionString(store)
};
ravenConnectionStrings.Add(ravenConnectionStr);
sqlConnectionStrings.Add(sqlConnectionStr);
var result1 = store.Maintenance.Send(new PutConnectionStringOperation <RavenConnectionString>(ravenConnectionStr));
Assert.NotNull(result1.RaftCommandIndex);
var result2 = store.Maintenance.Send(new PutConnectionStringOperation <SqlConnectionString>(sqlConnectionStr));
Assert.NotNull(result2.RaftCommandIndex);
}
var result = store.Maintenance.Send(new GetConnectionStringsOperation());
Assert.NotNull(result.SqlConnectionStrings);
Assert.NotNull(result.RavenConnectionStrings);
for (var i = 0; i < 5; i++)
{
result.SqlConnectionStrings.TryGetValue($"SqlConnectionString{i}", out var sql);
Assert.Equal(sql?.ConnectionString, sqlConnectionStrings[i].ConnectionString);
result.RavenConnectionStrings.TryGetValue($"RavenConnectionString{i}", out var raven);
Assert.Equal(raven?.TopologyDiscoveryUrls, ravenConnectionStrings[i].TopologyDiscoveryUrls);
Assert.Equal(raven?.Database, ravenConnectionStrings[i].Database);
}
}
}
public void CanGetAllConnectionStrings()
{
using (var store = GetDocumentStore())
{
var ravenConnectionStrings = new List <RavenConnectionString>();
var sqlConnectionStrings = new List <SqlConnectionString>();
for (var i = 0; i < 5; i++)
{
var ravenConnectionStr = new RavenConnectionString()
{
Name = $"RavenConnectionString{i}",
Url = $"http://127.0.0.1:808{i}",
Database = "Northwind",
};
var sqlConnectionStr = new SqlConnectionString
{
Name = $"SqlConnectionString{i}",
ConnectionString = SqlEtlTests.GetConnectionString(store)
};
ravenConnectionStrings.Add(ravenConnectionStr);
sqlConnectionStrings.Add(sqlConnectionStr);
store.Admin.Server.Send(new PutConnectionStringOperation <RavenConnectionString>(ravenConnectionStr, store.Database));
store.Admin.Server.Send(new PutConnectionStringOperation <SqlConnectionString>(sqlConnectionStr, store.Database));
}
var result = store.Admin.Server.Send(new GetConnectionStringsOperation(store.Database));
Assert.NotNull(result.SqlConnectionStrings);
Assert.NotNull(result.RavenConnectionStrings);
for (var i = 0; i < 5; i++)
{
result.SqlConnectionStrings.TryGetValue($"SqlConnectionString{i}", out var sql);
Assert.Equal(sql?.ConnectionString, sqlConnectionStrings[i].ConnectionString);
result.RavenConnectionStrings.TryGetValue($"RavenConnectionString{i}", out var raven);
Assert.Equal(raven?.Url, ravenConnectionStrings[i].Url);
Assert.Equal(raven?.Database, ravenConnectionStrings[i].Database);
}
}
}
public async Task AutoNamingAlgorithmOfOngoingTasksShouldTakeNameAlreadyExistsIntoAccount()
{
using (var store = GetDocumentStore())
{
var dbName = $"db/{Guid.NewGuid()}";
var csName = $"cs/{Guid.NewGuid()}";
var connectionString = new RavenConnectionString
{
Name = csName,
Database = dbName,
TopologyDiscoveryUrls = new[] { "http://127.0.0.1:12345" }
};
var result = await store.Maintenance.SendAsync(new PutConnectionStringOperation <RavenConnectionString>(connectionString));
Assert.NotNull(result.RaftCommandIndex);
await store.Maintenance.SendAsync(new UpdateExternalReplicationOperation(new ExternalReplication(dbName, csName)));
await store.Maintenance.SendAsync(new UpdateExternalReplicationOperation(new ExternalReplication(dbName, csName)));
var backupConfig = new PeriodicBackupConfiguration
{
LocalSettings = new LocalSettings
{
FolderPath = NewDataPath(suffix: "BackupFolder")
},
AzureSettings = new AzureSettings
{
StorageContainer = "abc"
},
FullBackupFrequency = "* */1 * * *",
IncrementalBackupFrequency = "* */2 * * *",
Disabled = true
};
await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfig));
await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfig));
var etlConfiguration = new RavenEtlConfiguration
{
ConnectionStringName = csName,
Transforms =
{
new Transformation()
{
Name = "loadAll",
Collections ={ "Users" },
Script = "loadToUsers(this)"
}
}
};
await store.Maintenance.SendAsync(new AddEtlOperation <RavenConnectionString>(etlConfiguration));
await store.Maintenance.SendAsync(new AddEtlOperation <RavenConnectionString>(etlConfiguration));
// for Pull Replication Hub name is required - no need to test
var sink = new PullReplicationAsSink
{
HubDefinitionName = "aa",
ConnectionString = connectionString,
ConnectionStringName = connectionString.Name
};
await store.Maintenance.SendAsync(new UpdatePullReplicationAsSinkOperation(sink));
await store.Maintenance.SendAsync(new UpdatePullReplicationAsSinkOperation(sink));
}
}
public async Task SkipExportingTheServerWideExternalReplication2()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
using (var store = GetDocumentStore())
{
var putConfiguration1 = new ServerWideExternalReplication
{
Name = "1",
Disabled = true,
TopologyDiscoveryUrls = new[] { store.Urls.First() }
};
var putConfiguration2 = new ServerWideExternalReplication
{
Name = "2",
Disabled = true,
TopologyDiscoveryUrls = new[] { store.Urls.First() }
};
await store.Maintenance.Server.SendAsync(new PutServerWideExternalReplicationOperation(putConfiguration1));
await store.Maintenance.Server.SendAsync(new PutServerWideExternalReplicationOperation(putConfiguration2));
var dbName = $"db/{Guid.NewGuid()}";
var csName = $"cs/{Guid.NewGuid()}";
var connectionString = new RavenConnectionString
{
Name = csName,
Database = dbName,
TopologyDiscoveryUrls = new[] { "http://127.0.0.1:12345" }
};
var result = await store.Maintenance.SendAsync(new PutConnectionStringOperation <RavenConnectionString>(connectionString));
Assert.NotNull(result.RaftCommandIndex);
await store.Maintenance.SendAsync(new UpdateExternalReplicationOperation(new ExternalReplication(dbName, csName)
{
Disabled = true
}));
var databaseRecord = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
Assert.Equal(3, databaseRecord.ExternalReplications.Count);
var serverWideBackupConfiguration = new ServerWideBackupConfiguration
{
Disabled = false,
FullBackupFrequency = "0 2 * * 0",
IncrementalBackupFrequency = "0 2 * * 1",
LocalSettings = new LocalSettings
{
FolderPath = backupPath
}
};
await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(serverWideBackupConfiguration));
var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
var backup = record.PeriodicBackups.First();
var backupTaskId = backup.TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
string backupDirectory = null;
var value = WaitForValue(() =>
{
var status = store.Maintenance.Send(new GetPeriodicBackupStatusOperation(backupTaskId)).Status;
backupDirectory = status?.LocalBackup.BackupDirectory;
return(status?.LastEtag);
}, 0);
Assert.Equal(0, value);
var files = Directory.GetFiles(backupDirectory)
.Where(BackupUtils.IsBackupFile)
.OrderBackups()
.ToArray();
var databaseName = GetDatabaseName() + "restore";
var restoreConfig = new RestoreBackupConfiguration
{
BackupLocation = backupDirectory,
DatabaseName = databaseName,
LastFileNameToRestore = files.OrderBackups().Last()
};
var restoreOperation = new RestoreBackupOperation(restoreConfig);
store.Maintenance.Server.Send(restoreOperation)
.WaitForCompletion(TimeSpan.FromSeconds(30));
// new server should have only 0 external replications
var server = GetNewServer();
using (Databases.EnsureDatabaseDeletion(databaseName, store))
using (var store2 = GetDocumentStore(new Options
{
CreateDatabase = false,
ModifyDatabaseName = s => databaseName,
Server = server
}))
{
store2.Maintenance.Server.Send(restoreOperation)
.WaitForCompletion(TimeSpan.FromSeconds(30));
var record2 = await store2.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName));
Assert.Equal(1, record2.ExternalReplications.Count);
}
}
}
public async Task CanExcludeForNewDatabase()
{
using (var store = GetDocumentStore())
{
var newDbName = store.Database + "-testDatabase";
var serverWideExternalReplication = new ServerWideExternalReplication
{
Disabled = true,
TopologyDiscoveryUrls = new[]
{
store.Urls.First()
},
ExcludedDatabases = new []
{
newDbName
}
};
var result = await store.Maintenance.Server.SendAsync(new PutServerWideExternalReplicationOperation(serverWideExternalReplication));
serverWideExternalReplication.Name = result.Name;
await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(newDbName)));
var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(newDbName));
Assert.Equal(0, record.ExternalReplications.Count);
var dbName = $"db/{Guid.NewGuid()}";
var csName = $"cs/{Guid.NewGuid()}";
var connectionString = new RavenConnectionString
{
Name = csName,
Database = dbName,
TopologyDiscoveryUrls = new[] { "http://127.0.0.1:12345" }
};
var putConnectionStringResult = await store.Maintenance.ForDatabase(newDbName).SendAsync(new PutConnectionStringOperation <RavenConnectionString>(connectionString));
Assert.NotNull(putConnectionStringResult.RaftCommandIndex);
var externalReplication = new ExternalReplication(dbName, csName)
{
Name = "Regular Task",
Disabled = true
};
await store.Maintenance.ForDatabase(newDbName).SendAsync(new UpdateExternalReplicationOperation(externalReplication));
record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
Assert.Equal(1, record.ExternalReplications.Count);
Assert.Equal(1, record.RavenConnectionStrings.Count);
serverWideExternalReplication.ExcludedDatabases = new[] { store.Database };
await store.Maintenance.Server.SendAsync(new PutServerWideExternalReplicationOperation(serverWideExternalReplication));
record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
Assert.Equal(0, record.ExternalReplications.Count);
Assert.Equal(0, record.RavenConnectionStrings.Count);
record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(newDbName));
Assert.Equal(2, record.ExternalReplications.Count);
Assert.Equal(2, record.RavenConnectionStrings.Count);
Assert.Equal(externalReplication.Name, record.ExternalReplications[0].Name);
Assert.Equal(PutServerWideExternalReplicationCommand.GetTaskName(serverWideExternalReplication.Name), record.ExternalReplications[1].Name);
serverWideExternalReplication.ExcludedDatabases = null;
await store.Maintenance.Server.SendAsync(new PutServerWideExternalReplicationOperation(serverWideExternalReplication));
using (Server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
var tasks = Server.ServerStore.Cluster.GetServerWideConfigurations(context, OngoingTaskType.Replication, serverWideExternalReplication.Name).ToList();
Assert.Equal(1, tasks.Count);
tasks[0].TryGet(nameof(ServerWideExternalReplication.ExcludedDatabases), out BlittableJsonReaderArray excludedDatabases);
Assert.NotNull(excludedDatabases);
Assert.Equal(0, excludedDatabases.Length);
}
var newDbName2 = store.Database + "-testDatabase2";
await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(newDbName2)));
record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(newDbName));
Assert.Equal(2, record.ExternalReplications.Count);
Assert.Equal(2, record.RavenConnectionStrings.Count);
record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(newDbName2));
Assert.Equal(1, record.ExternalReplications.Count);
Assert.Equal(1, record.RavenConnectionStrings.Count);
}
}
public async Task FailuresDuringEmptyQueueShouldCauseReelection()
{
var(nodes, leader) = await CreateRaftCluster(3);
var leaderTag = leader.ServerStore.NodeTag;
var dbName = GetDatabaseName();
var db = await CreateDatabaseInCluster(dbName, 3, leader.WebUrl);
var cmdCount = 0;
var exceptionThrown = new ManualResetEventSlim();
leader.ServerStore.Engine.BeforeAppendToRaftLog += (context, cmd) =>
{
if (++cmdCount > 10)
{
exceptionThrown.Set();
throw new DiskFullException("no more space");
}
};
var stateChanged = new ManualResetEventSlim();
leader.ServerStore.Engine.StateChanged += (_, transition) =>
{
if (transition.From == RachisState.Leader && transition.To == RachisState.Candidate)
{
stateChanged.Set();
}
};
var followers = nodes.Where(s => s != leader).ToList();
var leaderSteppedDown = leader.ServerStore.Engine.WaitForLeaveState(RachisState.Leader, CancellationToken.None);
var newLeaderElected = Task.WhenAny(followers.Select(s => s.ServerStore.WaitForState(RachisState.Leader, CancellationToken.None)));
var putConnectionStrings = Task.Run(async() =>
{
using (var store = new DocumentStore
{
Database = dbName,
Urls = db.Servers.Select(s => s.WebUrl).ToArray()
}.Initialize())
{
var urls = nodes.Select(s => s.WebUrl).ToArray();
for (int i = 0; i < 20; i++)
{
var cs = new RavenConnectionString
{
Database = $"db/{i}",
Name = $"cs/{i}",
TopologyDiscoveryUrls = urls
};
await store.Maintenance.SendAsync(new PutConnectionStringOperation <RavenConnectionString>(cs));
}
}
});
Assert.True(exceptionThrown.Wait(TimeSpan.FromSeconds(10)), $"no exception thrown. commands count : {cmdCount}");
Assert.True(stateChanged.Wait(TimeSpan.FromSeconds(15)),
await AddErrorMessageAndClusterDebugLogs(nodes,
new StringBuilder().AppendLine($"leader {leaderTag} did not have a state transition from 'Leader' to 'Candidate' after 15 seconds.")));
Assert.True(leaderSteppedDown.Wait(TimeSpan.FromSeconds(15)),
await AddErrorMessageAndClusterDebugLogs(nodes, new StringBuilder().AppendLine($"leader {leaderTag} did not step down after 15 seconds")));
Assert.True(newLeaderElected.Wait(TimeSpan.FromSeconds(15)),
await AddErrorMessageAndClusterDebugLogs(nodes, new StringBuilder().AppendLine($"old leader {leaderTag} stepped down, but no new leader was elected after 15 seconds")));
await putConnectionStrings;
}
public async Task EtlDestinationFailoverBetweenNodesWithinSameCluster()
{
var srcDb = "EtlDestinationFailoverBetweenNodesWithinSameClusterSrc";
var dstDb = "EtlDestinationFailoverBetweenNodesWithinSameClusterDst";
var(_, srcRaft) = await CreateRaftCluster(3, leaderIndex : 0);
var(_, dstRaft) = await CreateRaftCluster(3);
var srcNodes = await CreateDatabaseInCluster(srcDb, 3, srcRaft.WebUrl);
var destNode = await CreateDatabaseInCluster(dstDb, 3, dstRaft.WebUrl);
using (var src = new DocumentStore
{
Urls = new[] { srcRaft.WebUrl },
Database = srcDb,
Conventions = new DocumentConventions
{
DisableTopologyUpdates = true
}
}.Initialize())
using (var dest = new DocumentStore
{
Urls = destNode.Servers.Select(s => s.WebUrl).ToArray(),
Database = dstDb,
}.Initialize())
{
var myTag = srcRaft.ServerStore.NodeTag;
var connectionStringName = "EtlFailover";
var urls = new[] { "http://google.com", "http://localhost:1232", destNode.Servers[0].WebUrl };
var conflig = new RavenEtlConfiguration()
{
Name = connectionStringName,
ConnectionStringName = connectionStringName,
Transforms =
{
new Transformation
{
Name = $"ETL : {connectionStringName}",
Collections = new List <string>(new[] { "Users" }),
Script = null,
ApplyToAllDocuments = false,
Disabled = false
}
},
LoadRequestTimeoutInSec = 10,
MentorNode = myTag
};
var connectionString = new RavenConnectionString
{
Name = connectionStringName,
Database = dest.Database,
TopologyDiscoveryUrls = urls,
};
var result = src.Maintenance.Send(new PutConnectionStringOperation <RavenConnectionString>(connectionString));
Assert.NotNull(result.RaftCommandIndex);
var etlResult = src.Maintenance.Send(new AddEtlOperation <RavenConnectionString>(conflig));
var database = await srcNodes.Servers.Single(s => s.ServerStore.NodeTag == myTag)
.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(srcDb);
var etlDone = new ManualResetEventSlim();
database.EtlLoader.BatchCompleted += x =>
{
if (x.Statistics.LoadSuccesses > 0)
{
etlDone.Set();
}
};
using (var session = src.OpenSession())
{
session.Store(new User()
{
Name = "Joe Doe"
}, "users|");
session.SaveChanges();
}
Assert.True(etlDone.Wait(TimeSpan.FromMinutes(1)));
Assert.True(WaitForDocument <User>(dest, "users/1", u => u.Name == "Joe Doe", 30_000));
// BEFORE THE FIX: this will change the database record and restart the ETL, which would fail the test.
// we leave this here to make sure that such issue in future will fail the test immediately
await src.Maintenance.SendAsync(new PutClientConfigurationOperation(new ClientConfiguration()));
var taskInfo = (OngoingTaskRavenEtlDetails)src.Maintenance.Send(new GetOngoingTaskInfoOperation(etlResult.TaskId, OngoingTaskType.RavenEtl));
Assert.NotNull(taskInfo);
Assert.NotNull(taskInfo.DestinationUrl);
Assert.Equal(myTag, taskInfo.ResponsibleNode.NodeTag);
Assert.Null(taskInfo.Error);
Assert.Equal(OngoingTaskConnectionStatus.Active, taskInfo.TaskConnectionStatus);
etlDone.Reset();
DisposeServerAndWaitForFinishOfDisposal(destNode.Servers.Single(s => s.WebUrl == taskInfo.DestinationUrl));
using (var session = src.OpenSession())
{
session.Store(new User
{
Name = "Joe Doe2"
}, "users/2");
session.SaveChanges();
}
Assert.True(etlDone.Wait(TimeSpan.FromMinutes(1)));
Assert.True(WaitForDocument <User>(dest, "users/2", u => u.Name == "Joe Doe2", 30_000));
}
}
public AddEtl()
{
using (var store = new DocumentStore())
{
#region add_raven_etl
AddEtlOperation <RavenConnectionString> operation = new AddEtlOperation <RavenConnectionString>(
new RavenEtlConfiguration
{
ConnectionStringName = "raven-connection-string-name",
Name = "Employees ETL",
Transforms =
{
new Transformation
{
Name = "Script #1",
Collections =
{
"Employees"
},
Script = @"loadToEmployees ({
Name: this.FirstName + ' ' + this.LastName,
Title: this.Title
});"
}
}
});
AddEtlOperationResult result = store.Maintenance.Send(operation);
#endregion
}
using (var store = new DocumentStore())
#region raven_etl_connection_string
{
//define connection string
var ravenConnectionString = new RavenConnectionString()
{
//name connection string
Name = "raven-connection-string-name",
//define appropriate node
//Be sure that the node definition in the connection string has the "s" in https
TopologyDiscoveryUrls = new[] { "https://127.0.0.1:8080" },
//define database to connect with on the node
Database = "Northwind",
};
//create the connection string
var resultRavenString = store.Maintenance.Send(
new PutConnectionStringOperation <RavenConnectionString>(ravenConnectionString));
}
#endregion
{
using (var store = new DocumentStore())
{
#region add_sql_etl
AddEtlOperation <SqlConnectionString> operation = new AddEtlOperation <SqlConnectionString>(
new SqlEtlConfiguration
{
ConnectionStringName = "sql-connection-string-name",
FactoryName = "System.Data.SqlClient",
Name = "Orders to SQL",
SqlTables =
{
new SqlEtlTable {
TableName = "Orders", DocumentIdColumn = "Id", InsertOnlyMode = false
},
new SqlEtlTable {
TableName = "OrderLines", DocumentIdColumn = "OrderId", InsertOnlyMode = false
},
},
Transforms =
{
new Transformation
{
Name = "Script #1",
Collections =
{
"Orders"
},
Script = @"var orderData = {
Id: id(this),
OrderLinesCount: this.Lines.length,
TotalCost: 0
};
for (var i = 0; i < this.Lines.length; i++) {
var line = this.Lines[i];
orderData.TotalCost += line.PricePerUnit;
// Load to SQL table 'OrderLines'
loadToOrderLines({
OrderId: id(this),
Qty: line.Quantity,
Product: line.Product,
Cost: line.PricePerUnit
});
}
orderData.TotalCost = Math.round(orderData.TotalCost * 100) / 100;
// Load to SQL table 'Orders'
loadToOrders(orderData)"
}
}
});
AddEtlOperationResult result = store.Maintenance.Send(operation);
#endregion
}
using (var store = new DocumentStore())
#region sql_etl_connection_string
{
// define new connection string
PutConnectionStringOperation <SqlConnectionString> operation
= new PutConnectionStringOperation <SqlConnectionString>(
new SqlConnectionString
{
// name connection string
Name = "local_mysql",
// define FactoryName
FactoryName = "MySql.Data.MySqlClient",
// define database - may also need to define authentication and encryption parameters
// by default, encrypted databases are sent over encrypted channels
ConnectionString = "host=127.0.0.1;user=root;database=Northwind"
});
// create connection string
PutConnectionStringResult connectionStringResult
= store.Maintenance.Send(operation);
}
#endregion
}
}
public ConnectionStrings()
{
using (var store = new DocumentStore())
{
#region add_raven_connection_string
PutConnectionStringOperation <RavenConnectionString> operation
= new PutConnectionStringOperation <RavenConnectionString>(
new RavenConnectionString
{
Name = "raven2",
Database = "Northwind2",
TopologyDiscoveryUrls = new[]
{
// Be sure to include the "s" in https for secure servers.
"https://rvn2:8080"
}
});
PutConnectionStringResult connectionStringResult
= store.Maintenance.Send(operation);
#endregion
}
using (var store = new DocumentStore())
{
#region add_sql_connection_string
PutConnectionStringOperation <SqlConnectionString> operation
= new PutConnectionStringOperation <SqlConnectionString>(
new SqlConnectionString
{
Name = "local_mysql",
FactoryName = "MySql.Data.MySqlClient",
ConnectionString = "host=127.0.0.1;user=root;database=Northwind"
});
PutConnectionStringResult connectionStringResult
= store.Maintenance.Send(operation);
#endregion
}
#region get_all_connection_strings
using (var store = new DocumentStore())
{
GetConnectionStringsOperation operation = new GetConnectionStringsOperation();
GetConnectionStringsResult connectionStrings = store.Maintenance.Send(operation);
Dictionary <string, SqlConnectionString> sqlConnectionStrings = connectionStrings.SqlConnectionStrings;
Dictionary <string, RavenConnectionString> ravenConnectionStrings = connectionStrings.RavenConnectionStrings;
}
#endregion
#region get_connection_string_by_name
using (var store = new DocumentStore())
{
GetConnectionStringsOperation operation =
new GetConnectionStringsOperation("local_mysql", ConnectionStringType.Sql);
GetConnectionStringsResult connectionStrings = store.Maintenance.Send(operation);
Dictionary <string, SqlConnectionString> sqlConnectionStrings = connectionStrings.SqlConnectionStrings;
SqlConnectionString mysqlConnectionString = sqlConnectionStrings["local_mysql"];
}
#endregion
using (var store = new DocumentStore())
{
RavenConnectionString connectionString = null;
#region remove_raven_connection_string
RemoveConnectionStringOperation <RavenConnectionString> operation
= new RemoveConnectionStringOperation <RavenConnectionString>(
connectionString);
store.Maintenance.Send(operation);
#endregion
}
using (var store = new DocumentStore())
{
SqlConnectionString connectionString = null;
#region remove_sql_connection_string
RemoveConnectionStringOperation <SqlConnectionString> operation
= new RemoveConnectionStringOperation <SqlConnectionString>(
connectionString);
store.Maintenance.Send(operation);
#endregion
}
using (var store = new DocumentStore())
#region raven_etl_connection_string
{
//define connection string
var ravenConnectionString = new RavenConnectionString()
{
//name connection string
Name = "raven-connection-string-name",
//define appropriate node
//Be sure that the node definition in the connection string has the "s" in https
TopologyDiscoveryUrls = new[] { "https://127.0.0.1:8080" },
//define database to connect with on the node
Database = "Northwind",
};
//create the connection string
var resultRavenString = store.Maintenance.Send(
new PutConnectionStringOperation <RavenConnectionString>(ravenConnectionString));
}
#endregion
using (var store = new DocumentStore())
#region sql_etl_connection_string
{
// define new connection string
PutConnectionStringOperation <SqlConnectionString> operation
= new PutConnectionStringOperation <SqlConnectionString>(
new SqlConnectionString
{
// name connection string
Name = "local_mysql",
// define FactoryName
FactoryName = "MySql.Data.MySqlClient",
// define database - may also need to define authentication and encryption parameters
// by default, encrypted databases are sent over encrypted channels
ConnectionString = "host=127.0.0.1;user=root;database=Northwind"
});
// create connection string
PutConnectionStringResult connectionStringResult
= store.Maintenance.Send(operation);
}
#endregion
}
private bool ValidateConnectionString(DatabaseRecord newRecord, ExternalReplication externalReplication, out RavenConnectionString connectionString)
{
connectionString = null;
if (string.IsNullOrEmpty(externalReplication.ConnectionStringName))
{
var msg = $"The external replication {externalReplication.Name} to the database '{externalReplication.Database}' " +
"has an empty connection string name.";
if (_log.IsInfoEnabled)
{
_log.Info(msg);
}
_server.NotificationCenter.Add(AlertRaised.Create(
Database.Name,
"Connection string name is empty",
msg,
AlertType.Replication,
NotificationSeverity.Error));
return(false);
}
if (newRecord.RavenConnectionStrings.TryGetValue(externalReplication.ConnectionStringName, out connectionString) == false)
{
var msg = $"Could not find connection string with name {externalReplication.ConnectionStringName} " +
$"for the external replication task '{externalReplication.Name}' to '{externalReplication.Database}'.";
if (_log.IsInfoEnabled)
{
_log.Info(msg);
}
_server.NotificationCenter.Add(AlertRaised.Create(
Database.Name,
"Connection string not found",
msg,
AlertType.Replication,
NotificationSeverity.Error));
return(false);
}
return(true);
}
public ConnectionStrings()
{
using (var store = new DocumentStore())
{
#region add_raven_connection_string
PutConnectionStringOperation <RavenConnectionString> operation
= new PutConnectionStringOperation <RavenConnectionString>(
new RavenConnectionString
{
Name = "raven2",
Database = "Northwind2",
TopologyDiscoveryUrls = new[]
{
"http://rvn2:8080"
}
});
PutConnectionStringResult connectionStringResult
= store.Maintenance.Send(operation);
#endregion
}
using (var store = new DocumentStore())
{
#region add_sql_connection_string
PutConnectionStringOperation <SqlConnectionString> operation
= new PutConnectionStringOperation <SqlConnectionString>(
new SqlConnectionString
{
Name = "local_mysql",
FactoryName = "MySql.Data.MySqlClient",
ConnectionString = "host=127.0.0.1;user=root;database=Northwind"
});
PutConnectionStringResult connectionStringResult
= store.Maintenance.Send(operation);
#endregion
}
using (var store = new DocumentStore())
{
#region get_all_connection_strings
GetConnectionStringsOperation operation = new GetConnectionStringsOperation();
GetConnectionStringsResult connectionStrings = store.Maintenance.Send(operation);
Dictionary <string, SqlConnectionString> sqlConnectionStrings = connectionStrings.SqlConnectionStrings;
Dictionary <string, RavenConnectionString> ravenConnectionStrings = connectionStrings.RavenConnectionStrings;
#endregion
}
using (var store = new DocumentStore())
{
#region get_connection_string_by_name
GetConnectionStringsOperation operation =
new GetConnectionStringsOperation("local_mysql", ConnectionStringType.Sql);
GetConnectionStringsResult connectionStrings = store.Maintenance.Send(operation);
Dictionary <string, SqlConnectionString> sqlConnectionStrings = connectionStrings.SqlConnectionStrings;
SqlConnectionString mysqlConnectionString = sqlConnectionStrings["local_mysql"];
#endregion
}
using (var store = new DocumentStore())
{
RavenConnectionString connectionString = null;
#region remove_raven_connection_string
RemoveConnectionStringOperation <RavenConnectionString> operation
= new RemoveConnectionStringOperation <RavenConnectionString>(
connectionString);
store.Maintenance.Send(operation);
#endregion
}
using (var store = new DocumentStore())
{
SqlConnectionString connectionString = null;
#region remove_sql_connection_string
RemoveConnectionStringOperation <SqlConnectionString> operation
= new RemoveConnectionStringOperation <SqlConnectionString>(
connectionString);
store.Maintenance.Send(operation);
#endregion
}
}
public async Task ReplicateFromSingleSource()
{
var srcDb = "ReplicateFromSingleSourceSrc";
var dstDb = "ReplicateFromSingleSourceDst";
var(_, srcRaft) = await CreateRaftCluster(3);
var(_, dstRaft) = await CreateRaftCluster(1);
var srcNodes = await CreateDatabaseInCluster(srcDb, 3, srcRaft.WebUrl);
var destNode = await CreateDatabaseInCluster(dstDb, 1, dstRaft.WebUrl);
var node = srcNodes.Servers.First(x => x.ServerStore.NodeTag != srcRaft.ServerStore.NodeTag).ServerStore.NodeTag;
using (var src = new DocumentStore
{
Urls = srcNodes.Servers.Select(s => s.WebUrl).ToArray(),
Database = srcDb,
}.Initialize())
using (var dest = new DocumentStore
{
Urls = new[] { destNode.Servers[0].WebUrl },
Database = dstDb,
}.Initialize())
{
var connectionStringName = "EtlFailover";
var urls = new[] { destNode.Servers[0].WebUrl };
var config = new RavenEtlConfiguration()
{
Name = connectionStringName,
ConnectionStringName = connectionStringName,
Transforms =
{
new Transformation
{
Name = $"ETL : {connectionStringName}",
Collections = new List <string>(new[] { "Users" }),
Script = null,
ApplyToAllDocuments = false,
Disabled = false
}
},
LoadRequestTimeoutInSec = 30,
MentorNode = node
};
var connectionString = new RavenConnectionString
{
Name = connectionStringName,
Database = dest.Database,
TopologyDiscoveryUrls = urls,
};
var result = src.Maintenance.Send(new PutConnectionStringOperation <RavenConnectionString>(connectionString));
Assert.NotNull(result.RaftCommandIndex);
src.Maintenance.Send(new AddEtlOperation <RavenConnectionString>(config));
var originalTaskNode = srcNodes.Servers.Single(s => s.ServerStore.NodeTag == node);
using (var session = src.OpenSession())
{
session.Store(new User()
{
Name = "Joe Doe"
}, "users/1");
session.SaveChanges();
}
Assert.True(WaitForDocument <User>(dest, "users/1", u => u.Name == "Joe Doe", 30_000));
await ActionWithLeader((l) => l.ServerStore.RemoveFromClusterAsync(node));
await originalTaskNode.ServerStore.WaitForState(RachisState.Passive, CancellationToken.None);
using (var session = src.OpenSession())
{
session.Store(new User()
{
Name = "Joe Doe2"
}, "users/2");
session.SaveChanges();
}
Assert.True(WaitForDocument <User>(dest, "users/2", u => u.Name == "Joe Doe2", 30_000));
Assert.Throws <NodeIsPassiveException>(() =>
{
using (var originalSrc = new DocumentStore
{
Urls = new[] { originalTaskNode.WebUrl },
Database = srcDb,
Conventions = new DocumentConventions
{
DisableTopologyUpdates = true
}
}.Initialize())
{
using (var session = originalSrc.OpenSession())
{
session.Store(new User()
{
Name = "Joe Doe3"
}, "users/3");
session.SaveChanges();
}
}
});
}
}
public async Task WillWorkAfterResponsibleNodeRestart_RavenDB_13237()
{
var srcDb = "ETL-src";
var dstDb = "ETL-dst";
var(_, srcRaft) = await CreateRaftCluster(3, shouldRunInMemory : false);
var(_, dstRaft) = await CreateRaftCluster(1);
var srcNodes = await CreateDatabaseInCluster(srcDb, 2, srcRaft.WebUrl);
var destNode = await CreateDatabaseInCluster(dstDb, 1, dstRaft.WebUrl);
using (var src = new DocumentStore
{
Urls = srcNodes.Servers.Select(s => s.WebUrl).ToArray(),
Database = srcDb,
}.Initialize())
using (var dest = new DocumentStore
{
Urls = new[] { destNode.Servers[0].WebUrl },
Database = dstDb,
}.Initialize())
{
var name = "FailoverAfterRestart";
var urls = new[] { destNode.Servers[0].WebUrl };
var config = new RavenEtlConfiguration()
{
Name = name,
ConnectionStringName = name,
Transforms =
{
new Transformation
{
Name = $"ETL : {name}",
Collections = new List <string>(new[] { "Users" }),
Script = null,
ApplyToAllDocuments = false,
Disabled = false
}
},
LoadRequestTimeoutInSec = 30,
};
var connectionString = new RavenConnectionString
{
Name = name,
Database = dest.Database,
TopologyDiscoveryUrls = urls,
};
var result = src.Maintenance.Send(new PutConnectionStringOperation <RavenConnectionString>(connectionString));
Assert.NotNull(result.RaftCommandIndex);
src.Maintenance.Send(new AddEtlOperation <RavenConnectionString>(config));
var ongoingTask = src.Maintenance.Send(new GetOngoingTaskInfoOperation(name, OngoingTaskType.RavenEtl));
var responsibleNodeNodeTag = ongoingTask.ResponsibleNode.NodeTag;
var originalTaskNodeServer = srcNodes.Servers.Single(s => s.ServerStore.NodeTag == responsibleNodeNodeTag);
using (var session = src.OpenSession())
{
session.Store(new User()
{
Name = "Joe Doe"
}, "users/1");
session.SaveChanges();
}
Assert.True(WaitForDocument <User>(dest, "users/1", u => u.Name == "Joe Doe", 30_000));
var originalResult = DisposeServerAndWaitForFinishOfDisposal(originalTaskNodeServer);
using (var session = src.OpenSession())
{
session.Store(new User()
{
Name = "Joe Doe2"
}, "users/2");
session.SaveChanges();
}
Assert.True(WaitForDocument <User>(dest, "users/2", u => u.Name == "Joe Doe2", 30_000));
ongoingTask = src.Maintenance.Send(new GetOngoingTaskInfoOperation(name, OngoingTaskType.RavenEtl));
var currentNodeNodeTag = ongoingTask.ResponsibleNode.NodeTag;
var currentTaskNodeServer = srcNodes.Servers.Single(s => s.ServerStore.NodeTag == currentNodeNodeTag);
// start server which originally was handling ETL task
GetNewServer(new ServerCreationOptions
{
CustomSettings = new Dictionary <string, string>
{
{ RavenConfiguration.GetKey(x => x.Core.ServerUrls), originalResult.Url }
},
RunInMemory = false,
DeletePrevious = false,
DataDirectory = originalResult.DataDirectory
});
using (var store = new DocumentStore
{
Urls = new[] { originalResult.Url },
Database = srcDb,
Conventions =
{
DisableTopologyUpdates = true
}
}.Initialize())
{
using (var session = store.OpenSession())
{
session.Store(new User()
{
Name = "Joe Doe3"
}, "users/3");
session.SaveChanges();
}
Assert.True(WaitForDocument <User>(dest, "users/3", u => u.Name == "Joe Doe3", 30_000));
// force disposing second node to ensure the original node is reponsible for ETL task again
DisposeServerAndWaitForFinishOfDisposal(currentTaskNodeServer);
using (var session = store.OpenSession())
{
session.Store(new User()
{
Name = "Joe Doe4"
}, "users/4");
session.SaveChanges();
}
Assert.True(WaitForDocument <User>(dest, "users/4", u => u.Name == "Joe Doe4", 30_000));
}
}
}
public async Task ShouldSendCounterChangeMadeInCluster()
{
var srcDb = "13288-src";
var dstDb = "13288-dst";
var(_, srcRaft) = await CreateRaftCluster(2);
var(_, dstRaft) = await CreateRaftCluster(1);
var srcNodes = await CreateDatabaseInCluster(srcDb, 2, srcRaft.WebUrl);
var destNode = await CreateDatabaseInCluster(dstDb, 1, dstRaft.WebUrl);
using (var src = new DocumentStore
{
Urls = srcNodes.Servers.Select(s => s.WebUrl).ToArray(),
Database = srcDb,
}.Initialize())
using (var dest = new DocumentStore
{
Urls = new[] { destNode.Servers[0].WebUrl },
Database = dstDb,
}.Initialize())
{
var connectionStringName = "my-etl";
var urls = new[] { destNode.Servers[0].WebUrl };
var config = new RavenEtlConfiguration()
{
Name = connectionStringName,
ConnectionStringName = connectionStringName,
Transforms =
{
new Transformation
{
Name = $"ETL : {connectionStringName}",
Collections = new List <string>(new[] { "Users" }),
Script = null,
ApplyToAllDocuments = false,
Disabled = false
}
},
LoadRequestTimeoutInSec = 30,
MentorNode = "A"
};
var connectionString = new RavenConnectionString
{
Name = connectionStringName,
Database = dest.Database,
TopologyDiscoveryUrls = urls,
};
var result = src.Maintenance.Send(new PutConnectionStringOperation <RavenConnectionString>(connectionString));
Assert.NotNull(result.RaftCommandIndex);
src.Maintenance.Send(new AddEtlOperation <RavenConnectionString>(config));
var aNode = srcNodes.Servers.Single(s => s.ServerStore.NodeTag == "A");
var bNode = srcNodes.Servers.Single(s => s.ServerStore.NodeTag == "B");
// modify counter on A node (mentor of ETL task)
using (var aSrc = new DocumentStore
{
Urls = new[] { aNode.WebUrl },
Database = srcDb,
Conventions = new DocumentConventions
{
DisableTopologyUpdates = true
}
}.Initialize())
{
using (var session = aSrc.OpenSession())
{
session.Store(new User()
{
Name = "Joe Doe"
}, "users/1");
session.CountersFor("users/1").Increment("likes");
session.Advanced.WaitForReplicationAfterSaveChanges();
session.SaveChanges();
}
}
Assert.True(WaitForDocument <User>(dest, "users/1", u => u.Name == "Joe Doe", 30_000));
using (var session = dest.OpenSession())
{
var user = session.Load <User>("users/1");
Assert.NotNull(user);
Assert.Equal("Joe Doe", user.Name);
var counter = session.CountersFor("users/1").Get("likes");
Assert.NotNull(counter);
Assert.Equal(1, counter.Value);
}
// modify counter on B node (not mentor)
using (var bSrc = new DocumentStore
{
Urls = new[] { bNode.WebUrl },
Database = srcDb,
Conventions = new DocumentConventions
{
DisableTopologyUpdates = true
}
}.Initialize())
{
using (var session = bSrc.OpenSession())
{
session.CountersFor("users/1").Increment("likes");
session.SaveChanges();
}
}
Assert.True(Replication.WaitForCounterReplication(new List <IDocumentStore>
{
dest
}, "users/1", "likes", 2, TimeSpan.FromSeconds(60)));
}
}
public async Task RavenEtlWithTimeSeries_WhenEtlNodeTryToProcessTimeSeriesWithoutDocAndTheEtlMovesToAnotherNodeBeforeTheDocProcessed()
{
string connectionStringName = Context.MethodName;
var time = new DateTime(2020, 04, 27);
const string timeSeriesName = "Heartrate";
const string tag = "fitbit";
const double value = 58d;
var tsOwnerId = "users/1";
var justCheckEtl = new User();
const int clusterSize = 3;
(_, RavenServer leader) = await CreateRaftCluster(clusterSize, customSettings : _customSettings);
using var src = GetDocumentStore(new Options { Server = leader, ReplicationFactor = clusterSize });
var dest = GetDocumentStore();
var etlConfiguration = new RavenEtlConfiguration
{
Name = connectionStringName,
ConnectionStringName = connectionStringName,
Transforms = { new Transformation {
Name = $"ETL : {connectionStringName}", ApplyToAllDocuments = true
} },
MentorNode = "A",
};
var connectionString = new RavenConnectionString
{
Name = connectionStringName,
Database = dest.Database,
TopologyDiscoveryUrls = dest.Urls,
};
Assert.NotNull(src.Maintenance.Send(new PutConnectionStringOperation <RavenConnectionString>(connectionString)));
var etlResult = src.Maintenance.Send(new AddEtlOperation <RavenConnectionString>(etlConfiguration));
var srcDatabase = await GetDatabase(leader, src.Database);
using (var context = DocumentsOperationContext.ShortTermSingleUse(srcDatabase))
using (var tr = context.OpenWriteTransaction())
{
var tsStorage = srcDatabase.DocumentsStorage.TimeSeriesStorage;
var toAppend = new[]
{
new SingleResult
{
Status = TimeSeriesValuesSegment.Live,
Tag = context.GetLazyString(tag),
Timestamp = time,
Type = SingleResultType.Raw,
Values = new Memory <double>(new [] { value })
},
};
tsStorage.AppendTimestamp(context, tsOwnerId, "Users", timeSeriesName.ToLower(), toAppend, EtlTimeSeriesTests.AppendOptionsForEtlTest);
tr.Commit();
}
using (var session = src.OpenAsyncSession())
{
await session.StoreAsync(justCheckEtl);
await session.SaveChangesAsync();
}
await WaitForNotNullAsync(async() =>
{
using var session = dest.OpenAsyncSession();
return(await session.LoadAsync <User>(justCheckEtl.Id));
}, interval : _waitInterval);
etlConfiguration.MentorNode = "B";
src.Maintenance.Send(new UpdateEtlOperation <RavenConnectionString>(etlResult.TaskId, etlConfiguration));
using (var context = DocumentsOperationContext.ShortTermSingleUse(srcDatabase))
using (var tr = context.OpenWriteTransaction())
{
var ab = new DynamicJsonValue
{
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Users",
[Constants.Documents.Metadata.TimeSeries] = new DynamicJsonArray(new [] { timeSeriesName }),
}
};
using var doc = context.ReadObject(ab, tsOwnerId, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
srcDatabase.DocumentsStorage.Put(context, tsOwnerId, null, doc, flags: DocumentFlags.HasTimeSeries);
tr.Commit();
}
await AssertWaitForNotNullAsync(async() =>
{
using var session = dest.OpenAsyncSession();
var timeSeriesEntries = await session.TimeSeriesFor(tsOwnerId, timeSeriesName).GetAsync(time, time);
return(timeSeriesEntries?.FirstOrDefault());
}, interval : 1000);
}
