public void GetSqlEtlTaskInfo()
{
var sqlScript = @"
var orderData = {
Id: __document_id,
OrderLinesCount: this.OrderLines.length,
TotalCost: 0
};
loadToOrders(orderData);
";
var sqlConfiguration = new SqlEtlConfiguration()
{
Name = "abc",
ConnectionStringName = "abc",
SqlTables =
{
new SqlEtlTable {
TableName = "Orders", DocumentIdColumn = "Id", InsertOnlyMode = false
},
new SqlEtlTable {
TableName = "OrderLines", DocumentIdColumn = "OrderId", InsertOnlyMode = false
},
},
Transforms =
{
new Transformation()
{
Name = "OrdersAndLines",
Collections = new List <string>{
"Orders"
},
Script = sqlScript
}
}
};
using (var store = GetDocumentStore())
{
var sqlConnectionString = new SqlConnectionString
{
Name = "abc",
ConnectionString = @"Data Source=localhost\sqlexpress;Integrated Security=SSPI;Connection Timeout=3" + $";Initial Catalog=SqlReplication-{store.Database};",
FactoryName = "System.Data.SqlClient"
};
var result = store.Maintenance.Send(new PutConnectionStringOperation <SqlConnectionString>(sqlConnectionString));
Assert.NotNull(result.RaftCommandIndex);
var sqlEtlResult = store.Maintenance.Send(new AddEtlOperation <SqlConnectionString>(sqlConfiguration));
var taskId = sqlEtlResult.TaskId;
var op = new GetOngoingTaskInfoOperation(taskId, OngoingTaskType.SqlEtl);
var sqlResult = (OngoingTaskSqlEtlDetails)store.Maintenance.Send(op);
Assert.Equal("abc", sqlResult.Configuration.ConnectionStringName);
Assert.Equal("abc", sqlResult.Configuration.Name);
Assert.Equal("OrdersAndLines", sqlResult.Configuration.Transforms[0].Name);
Assert.Equal(sqlScript, sqlResult.Configuration.Transforms[0].Script);
Assert.Equal("Orders", sqlResult.Configuration.Transforms[0].Collections[0]);
Assert.NotNull(sqlResult.Configuration.SqlTables);
Assert.Equal(sqlConfiguration.Name, sqlResult?.TaskName);
op = new GetOngoingTaskInfoOperation("abc", OngoingTaskType.SqlEtl);
sqlResult = (OngoingTaskSqlEtlDetails)store.Maintenance.Send(op);
Assert.Equal("abc", sqlResult.Configuration.ConnectionStringName);
Assert.Equal("abc", sqlResult.Configuration.Name);
Assert.Equal("OrdersAndLines", sqlResult.Configuration.Transforms[0].Name);
Assert.Equal(sqlScript, sqlResult.Configuration.Transforms[0].Script);
Assert.Equal("Orders", sqlResult.Configuration.Transforms[0].Collections[0]);
Assert.NotNull(sqlResult.Configuration.SqlTables);
Assert.Equal(taskId, sqlResult.TaskId);
}
}
public SqlDocumentTransformer(Transformation transformation, DocumentDatabase database, DocumentsOperationContext context, SqlEtlConfiguration config)
: base(database, context, new PatchRequest(transformation.Script, PatchRequestType.SqlEtl))
{
_transformation = transformation;
_config = config;
_tables = new Dictionary <string, SqlTableWithRecords>(_config.SqlTables.Count);
var tables = new string[config.SqlTables.Count];
for (var i = 0; i < config.SqlTables.Count; i++)
{
tables[i] = config.SqlTables[i].TableName;
}
LoadToDestinations = tables;
}
private IEnumerable <EtlProcess> GetRelevantProcesses <T, TConnectionString>(List <T> configurations, HashSet <string> uniqueNames) where T : EtlConfiguration <TConnectionString> where TConnectionString : ConnectionString
{
foreach (var config in configurations)
{
SqlEtlConfiguration sqlConfig = null;
RavenEtlConfiguration ravenConfig = null;
var connectionStringNotFound = false;
switch (config.EtlType)
{
case EtlType.Raven:
ravenConfig = config as RavenEtlConfiguration;
if (_databaseRecord.RavenConnectionStrings.TryGetValue(config.ConnectionStringName, out var ravenConnection))
{
ravenConfig.Initialize(ravenConnection);
}
else
{
connectionStringNotFound = true;
}
break;
case EtlType.Sql:
sqlConfig = config as SqlEtlConfiguration;
if (_databaseRecord.SqlConnectionStrings.TryGetValue(config.ConnectionStringName, out var sqlConnection))
{
sqlConfig.Initialize(sqlConnection);
}
else
{
connectionStringNotFound = true;
}
break;
default:
ThrownUnknownEtlConfiguration(config.GetType());
break;
}
if (connectionStringNotFound)
{
LogConfigurationError(config,
new List <string>
{
$"Connection string named '{config.ConnectionStringName}' was not found for {config.EtlType} ETL"
});
continue;
}
if (ValidateConfiguration(config, uniqueNames) == false)
{
continue;
}
if (config.Disabled)
{
continue;
}
if (_databaseRecord.Topology.WhoseTaskIsIt(config, _serverStore.IsPassive()) != _serverStore.NodeTag)
{
continue;
}
foreach (var transform in config.Transforms)
{
if (transform.Disabled)
{
continue;
}
EtlProcess process = null;
if (sqlConfig != null)
{
process = new SqlEtl(transform, sqlConfig, _database, _serverStore);
}
if (ravenConfig != null)
{
process = new RavenEtl(transform, ravenConfig, _database, _serverStore);
}
yield return(process);
}
}
}
public async Task CanGetTaskInfo()
{
var clusterSize = 3;
var databaseName = "TestDB";
var leader = await CreateRaftClusterAndGetLeader(clusterSize);
ModifyOngoingTaskResult addWatcherRes;
UpdatePeriodicBackupOperationResult updateBackupResult;
AddEtlOperationResult addRavenEtlResult;
AddEtlOperationResult addSqlEtlResult;
RavenEtlConfiguration etlConfiguration;
SqlEtlConfiguration sqlConfiguration;
ExternalReplication watcher;
SqlConnectionString sqlConnectionString;
var sqlScript = @"
var orderData = {
Id: __document_id,
OrderLinesCount: this.OrderLines.length,
TotalCost: 0
};
loadToOrders(orderData);
";
using (var store = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName
}.Initialize())
{
var doc = new DatabaseRecord(databaseName);
var databaseResult = await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(doc, clusterSize));
Assert.Equal(clusterSize, databaseResult.Topology.AllNodes.Count());
foreach (var server in Servers)
{
await server.ServerStore.Cluster.WaitForIndexNotification(databaseResult.RaftCommandIndex);
}
foreach (var server in Servers)
{
await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName);
}
watcher = new ExternalReplication("Watcher1", "Connection")
{
Name = "MyExternalReplication"
};
addWatcherRes = await AddWatcherToReplicationTopology((DocumentStore)store, watcher, new[] { "http://127.0.0.1:9090" });
var backupConfig = new PeriodicBackupConfiguration
{
Name = "backup1",
LocalSettings = new LocalSettings
{
FolderPath = NewDataPath(suffix: "BackupFolder")
},
AzureSettings = new AzureSettings
{
StorageContainer = "abc"
},
FullBackupFrequency = "* */1 * * *",
IncrementalBackupFrequency = "* */2 * * *",
Disabled = true
};
updateBackupResult = await store.Maintenance.Server.SendAsync(new UpdatePeriodicBackupOperation(backupConfig, store.Database));
store.Maintenance.Server.Send(new PutConnectionStringOperation <RavenConnectionString>(new RavenConnectionString
{
Name = "cs",
TopologyDiscoveryUrls = new [] { "http://127.0.0.1:8080" },
Database = "Northwind",
}, store.Database));
etlConfiguration = new RavenEtlConfiguration()
{
Name = "tesst",
ConnectionStringName = "cs",
Transforms =
{
new Transformation()
{
Name = "loadAll",
Collections ={ "Users" },
Script = "loadToUsers(this)"
}
}
};
addRavenEtlResult = store.Maintenance.Server.Send(new AddEtlOperation <RavenConnectionString>(etlConfiguration, store.Database));
sqlConnectionString = new SqlConnectionString
{
Name = "abc",
ConnectionString = @"Data Source=localhost\sqlexpress;Integrated Security=SSPI;Connection Timeout=3" + $";Initial Catalog=SqlReplication-{store.Database};"
};
store.Maintenance.Server.Send(new PutConnectionStringOperation <SqlConnectionString>(sqlConnectionString, store.Database));
sqlConfiguration = new SqlEtlConfiguration()
{
Name = "abc",
ConnectionStringName = "abc",
FactoryName = "System.Data.SqlClient",
SqlTables =
{
new SqlEtlTable {
TableName = "Orders", DocumentIdColumn = "Id", InsertOnlyMode = false
},
new SqlEtlTable {
TableName = "OrderLines", DocumentIdColumn = "OrderId", InsertOnlyMode = false
},
},
Transforms =
{
new Transformation()
{
Name = "OrdersAndLines",
Collections = new List <string>{
"Orders"
},
Script = sqlScript
}
}
};
addSqlEtlResult = store.Maintenance.Server.Send(new AddEtlOperation <SqlConnectionString>(sqlConfiguration, store.Database));
}
using (var store = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName,
Conventions =
{
DisableTopologyUpdates = true
}
}.Initialize())
{
var taskId = addWatcherRes.TaskId;
var replicationResult = (OngoingTaskReplication) await GetTaskInfo((DocumentStore)store, taskId, OngoingTaskType.Replication);
Assert.Equal(watcher.Database, replicationResult.DestinationDatabase);
Assert.Equal(watcher.Url, replicationResult.DestinationUrl);
Assert.Equal(watcher.Name, replicationResult.TaskName);
taskId = updateBackupResult.TaskId;
var backupResult = (OngoingTaskBackup) await GetTaskInfo((DocumentStore)store, taskId, OngoingTaskType.Backup);
Assert.Equal("Local", backupResult.BackupDestinations[0]);
Assert.Equal("Azure", backupResult.BackupDestinations[1]);
Assert.Equal("backup1", backupResult.TaskName);
Assert.Equal(OngoingTaskState.Disabled, backupResult.TaskState);
taskId = addRavenEtlResult.TaskId;
var etlResult = (OngoingTaskRavenEtlDetails) await GetTaskInfo((DocumentStore)store, taskId, OngoingTaskType.RavenEtl);
Assert.Equal("cs", etlResult.Configuration.ConnectionStringName);
Assert.Equal("tesst", etlResult.Configuration.Name);
Assert.Equal("loadAll", etlResult.Configuration.Transforms[0].Name);
Assert.Equal("loadToUsers(this)", etlResult.Configuration.Transforms[0].Script);
Assert.Equal("Users", etlResult.Configuration.Transforms[0].Collections[0]);
Assert.Equal(etlConfiguration.Name, etlResult?.TaskName);
taskId = addSqlEtlResult.TaskId;
var sqlResult = (OngoingTaskSqlEtlDetails) await GetTaskInfo((DocumentStore)store, taskId, OngoingTaskType.SqlEtl);
Assert.Equal("abc", sqlResult.Configuration.ConnectionStringName);
Assert.Equal("abc", sqlResult.Configuration.Name);
Assert.Equal("OrdersAndLines", sqlResult.Configuration.Transforms[0].Name);
Assert.Equal(sqlScript, sqlResult.Configuration.Transforms[0].Script);
Assert.Equal("Orders", sqlResult.Configuration.Transforms[0].Collections[0]);
Assert.NotNull(sqlResult.Configuration.SqlTables);
Assert.Equal(sqlConfiguration.Name, sqlResult?.TaskName);
}
}
public void HandleDatabaseRecordChange(DatabaseRecord record)
{
if (record == null)
{
return;
}
var myRavenEtl = new List <RavenEtlConfiguration>();
var mySqlEtl = new List <SqlEtlConfiguration>();
foreach (var config in record.RavenEtls)
{
if (IsMyEtlTask <RavenEtlConfiguration, RavenConnectionString>(record, config))
{
myRavenEtl.Add(config);
}
}
foreach (var config in record.SqlEtls)
{
if (IsMyEtlTask <SqlEtlConfiguration, SqlConnectionString>(record, config))
{
mySqlEtl.Add(config);
}
}
var toRemove = _processes.GroupBy(x => x.ConfigurationName).ToDictionary(x => x.Key, x => x.ToList());
foreach (var processesPerConfig in _processes.GroupBy(x => x.ConfigurationName))
{
var process = processesPerConfig.First();
Debug.Assert(processesPerConfig.All(x => x.GetType() == process.GetType()));
if (process is RavenEtl ravenEtl)
{
RavenEtlConfiguration existing = null;
foreach (var config in myRavenEtl)
{
if (ravenEtl.Configuration.IsEqual(config))
{
existing = config;
break;
}
}
if (existing != null)
{
toRemove.Remove(processesPerConfig.Key);
myRavenEtl.Remove(existing);
}
}
else if (process is SqlEtl sqlEtl)
{
SqlEtlConfiguration existing = null;
foreach (var config in mySqlEtl)
{
if (sqlEtl.Configuration.IsEqual(config))
{
existing = config;
break;
}
}
if (existing != null)
{
toRemove.Remove(processesPerConfig.Key);
mySqlEtl.Remove(existing);
}
}
else
{
throw new InvalidOperationException($"Unknown ETL process type: {process.GetType()}");
}
}
Parallel.ForEach(toRemove, x =>
{
foreach (var process in x.Value)
{
try
{
process.Stop();
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info($"Failed to stop ETL process {process.Name} on the database record change", e);
}
}
}
});
LoadProcesses(record, myRavenEtl, mySqlEtl, toRemove.SelectMany(x => x.Value).ToList());
// unsubscribe old etls _after_ we start new processes to ensure the tombstone cleaner
// constantly keeps track of tombstones processed by ETLs so it won't delete them during etl processes reloading
foreach (var processesPerConfig in toRemove)
{
foreach (var process in processesPerConfig.Value)
{
_database.TombstoneCleaner.Unsubscribe(process);
}
}
Parallel.ForEach(toRemove, x =>
{
foreach (var process in x.Value)
{
try
{
process.Dispose();
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info($"Failed to dispose ETL process {process.Name} on the database record change", e);
}
}
}
});
}
public SqlDocumentTransformer(Transformation transformation, DocumentDatabase database, DocumentsOperationContext context, SqlEtlConfiguration config)
: base(database, context, new PatchRequest(transformation.Script, PatchRequestType.SqlEtl), null)
{
_transformation = transformation;
_config = config;
var destinationTables = transformation.GetCollectionsFromScript();
LoadToDestinations = destinationTables;
_tables = new Dictionary <string, SqlTableWithRecords>(destinationTables.Length);
_tablesForScript = new List <SqlEtlTable>(destinationTables.Length);
// ReSharper disable once ForCanBeConvertedToForeach
for (var i = 0; i < _config.SqlTables.Count; i++)
{
var table = _config.SqlTables[i];
if (destinationTables.Contains(table.TableName, StringComparer.OrdinalIgnoreCase))
{
_tablesForScript.Add(table);
}
}
if (_transformation.IsLoadingAttachments)
{
_loadedAttachments = new Dictionary <string, Queue <Attachment> >(StringComparer.OrdinalIgnoreCase);
}
}
public void HandleDatabaseRecordChange(DatabaseRecord record)
{
if (record == null)
{
return;
}
var myRavenEtl = new List <RavenEtlConfiguration>();
var mySqlEtl = new List <SqlEtlConfiguration>();
foreach (var config in record.RavenEtls)
{
if (IsMyEtlTask <RavenEtlConfiguration, RavenConnectionString>(record, config))
{
myRavenEtl.Add(config);
}
}
foreach (var config in record.SqlEtls)
{
if (IsMyEtlTask <SqlEtlConfiguration, SqlConnectionString>(record, config))
{
mySqlEtl.Add(config);
}
}
var toRemove = new List <EtlProcess>(_processes);
foreach (var etlProcess in _processes)
{
if (etlProcess is RavenEtl ravenEtl)
{
RavenEtlConfiguration existing = null;
foreach (var config in myRavenEtl)
{
if (ravenEtl.Configuration.IsEqual(config))
{
existing = config;
break;
}
}
if (existing != null)
{
toRemove.Remove(etlProcess);
myRavenEtl.Remove(existing);
}
}
if (etlProcess is SqlEtl sqlEtl)
{
SqlEtlConfiguration existing = null;
foreach (var config in mySqlEtl)
{
if (sqlEtl.Configuration.IsEqual(config))
{
existing = config;
break;
}
}
if (existing != null)
{
toRemove.Remove(etlProcess);
mySqlEtl.Remove(existing);
}
}
}
Parallel.ForEach(toRemove, x =>
{
try
{
x.Stop();
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info($"Failed to stop ETL process {x.Name} on the database record change", e);
}
}
});
LoadProcesses(record, myRavenEtl, mySqlEtl, toRemove);
// unsubscribe old etls _after_ we start new processes to ensure the tombstone cleaner
// constantly keeps track of tombstones processed by ETLs so it won't delete them during etl processes reloading
foreach (var process in toRemove)
{
_database.TombstoneCleaner.Unsubscribe(process);
}
Parallel.ForEach(toRemove, x =>
{
try
{
x.Dispose();
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info($"Failed to dispose ETL process {x.Name} on the database record change", e);
}
}
});
}
