public async Task CanDeleteEtl()
{
using (var store = GetDocumentStore())
{
var database = await GetDatabase(store.Database);
var configuration = new RavenEtlConfiguration()
{
ConnectionStringName = "test",
Name = "aaa",
Transforms =
{
new Transformation()
{
Collections ={ "Users" }
}
}
};
var result = AddEtl(store, configuration, new RavenConnectionString()
{
Name = "test",
Url = "http://127.0.0.1:8080",
Database = "Northwind",
});
store.Admin.Server.Send(new DeleteOngoingTaskOperation(database.Name, result.TaskId, OngoingTaskType.RavenEtl));
}
}
public void Error_if_script_does_not_contain_any_loadTo_method()
{
var config = new RavenEtlConfiguration
{
Name = "test",
ConnectionStringName = "test",
Transforms =
{
new Transformation
{
Name = "test",
Collections ={ "Users" },
Script = @"this.Name = 'aaa';"
}
}
};
config.Initialize(new RavenConnectionString()
{
Database = "Foo", TopologyDiscoveryUrls = new[] { "http://localhost:8080" }
});
List <string> errors;
config.Validate(out errors);
Assert.Equal(1, errors.Count);
Assert.Equal("No `loadTo[CollectionName]` method call found in 'test' script", errors[0]);
}
public void Can_put_space_after_loadTo_method_in_script()
{
var config = new RavenEtlConfiguration
{
Name = "test",
ConnectionStringName = "test",
Transforms =
{
new Transformation
{
Name = "test",
Collections ={ "Users" },
Script = @"loadToUsers (this);"
}
}
};
config.Initialize(new RavenConnectionString()
{
Database = "Foo", TopologyDiscoveryUrls = new [] { "http://localhost:8080" }
});
List <string> errors;
config.Validate(out errors);
Assert.Equal(0, errors.Count);
var collections = config.Transforms[0].GetCollectionsFromScript();
Assert.Equal(1, collections.Length);
Assert.Equal("Users", collections[0]);
}
public void CanDeleteEtl()
{
using (var store = GetDocumentStore())
{
var configuration = new RavenEtlConfiguration
{
ConnectionStringName = "test",
Name = "aaa",
Transforms =
{
new Transformation
{
Name = "S1",
Collections ={ "Users" }
}
}
};
var result = AddEtl(store, configuration, new RavenConnectionString
{
Name = "test",
TopologyDiscoveryUrls = new[] { "http://127.0.0.1:8080" },
Database = "Northwind",
});
store.Maintenance.Send(new DeleteOngoingTaskOperation(result.TaskId, OngoingTaskType.RavenEtl));
var ongoingTask = store.Maintenance.Send(new GetOngoingTaskInfoOperation(result.TaskId, OngoingTaskType.RavenEtl));
Assert.Null(ongoingTask);
}
}
public async Task CanDisableEtl()
{
using (var store = GetDocumentStore())
{
var database = await GetDatabase(store.Database);
var configuration = new RavenEtlConfiguration()
{
ConnectionStringName = "test",
Name = "aaa",
Transforms =
{
new Transformation()
{
Collections ={ "Users" }
}
}
};
var result = AddEtl(store, configuration, new RavenConnectionString()
{
Name = "test",
Url = "http://127.0.0.1:8080",
Database = "Northwind",
});
store.Admin.Server.Send(new ToggleTaskStateOperation(database.Name, result.TaskId, OngoingTaskType.RavenEtl, true));
var ongoingTask = store.Admin.Server.Send(new GetOngoingTaskInfoOperation(store.Database, result.TaskId, OngoingTaskType.RavenEtl));
Assert.Equal(OngoingTaskState.Disabled, ongoingTask.TaskState);
}
}
public void CanDisableEtl()
{
using (var store = GetDocumentStore())
{
var configuration = new RavenEtlConfiguration
{
ConnectionStringName = "test",
Name = "aaa",
Transforms =
{
new Transformation
{
Name = "S1",
Collections ={ "Users" }
}
}
};
var result = AddEtl(store, configuration, new RavenConnectionString
{
Name = "test",
TopologyDiscoveryUrls = new[] { "http://127.0.0.1:8080" },
Database = "Northwind",
});
var toggleResult = store.Maintenance.Send(new ToggleOngoingTaskStateOperation(result.TaskId, OngoingTaskType.RavenEtl, true));
Assert.NotNull(toggleResult);
Assert.True(toggleResult.RaftCommandIndex > 0);
Assert.True(toggleResult.TaskId > 0);
var ongoingTask = store.Maintenance.Send(new GetOngoingTaskInfoOperation(result.TaskId, OngoingTaskType.RavenEtl));
Assert.Equal(OngoingTaskState.Disabled, ongoingTask.TaskState);
}
}
public void Error_if_script_has_both_apply_to_all_documents_and_collections_specified()
{
var config = new RavenEtlConfiguration
{
Name = "test",
ConnectionStringName = "test",
Transforms =
{
new Transformation
{
Name = "test",
ApplyToAllDocuments = true,
Collections = { "Users" }
}
}
};
config.Initialize(new RavenConnectionString()
{
Database = "Foo", TopologyDiscoveryUrls = new[] { "http://localhost:8080" }
});
List <string> errors;
config.Validate(out errors);
Assert.Equal(1, errors.Count);
Assert.Equal("Collections cannot be specified when ApplyToAllDocuments is set", errors[0]);
}
internal static async Task <AddEtlOperationResult> AddEtl(IDocumentStore source, string destination, string[] urls, string mentor)
{
var connectionStringName = $"RavenEtl_From{source.Database}_To{destination}";
var config = new RavenEtlConfiguration()
{
Name = connectionStringName,
ConnectionStringName = connectionStringName,
LoadRequestTimeoutInSec = 10,
MentorNode = mentor,
Transforms = new List <Transformation>
{
new Transformation
{
Name = $"ETL : {connectionStringName}",
ApplyToAllDocuments = true,
IsEmptyScript = true
}
}
};
var connectionString = new RavenConnectionString
{
Name = connectionStringName,
Database = destination,
TopologyDiscoveryUrls = urls,
};
var result = await source.Maintenance.SendAsync(new PutConnectionStringOperation <RavenConnectionString>(connectionString));
Assert.NotNull(result.RaftCommandIndex);
return(await source.Maintenance.SendAsync(new AddEtlOperation <RavenConnectionString>(config)));
}
public async Task CanUpdateEtl()
{
using (var store = GetDocumentStore())
{
var database = await GetDatabase(store.Database);
var configuration = new RavenEtlConfiguration()
{
ConnectionStringName = "test",
Name = "aaa",
Transforms =
{
new Transformation()
{
Collections ={ "Users" }
}
}
};
var result = AddEtl(store, configuration, new RavenConnectionString()
{
Name = "test",
Url = "http://127.0.0.1:8080",
Database = "Northwind",
});
configuration.Transforms[0].Disabled = true;
store.Admin.Server.Send(new UpdateEtlOperation <RavenConnectionString>(result.TaskId, configuration, database.Name));
}
}
public async Task AutoNamingAlgorithmOfOngoingTasksShouldTakeNameAlreadyExistsIntoAccount()
{
using (var store = GetDocumentStore())
{
var dbName = $"db/{Guid.NewGuid()}";
var csName = $"cs/{Guid.NewGuid()}";
await store.Maintenance.SendAsync(new PutConnectionStringOperation <RavenConnectionString>(new RavenConnectionString
{
Name = csName,
Database = dbName,
TopologyDiscoveryUrls = new[] { "http://127.0.0.1:12345" }
}));
await store.Maintenance.SendAsync(new UpdateExternalReplicationOperation(new ExternalReplication(dbName, csName)));
await store.Maintenance.SendAsync(new UpdateExternalReplicationOperation(new ExternalReplication(dbName, csName)));
var backupConfig = new PeriodicBackupConfiguration
{
LocalSettings = new LocalSettings
{
FolderPath = NewDataPath(suffix: "BackupFolder")
},
AzureSettings = new AzureSettings
{
StorageContainer = "abc"
},
FullBackupFrequency = "* */1 * * *",
IncrementalBackupFrequency = "* */2 * * *",
Disabled = true
};
await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfig));
await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfig));
var etlConfiguration = new RavenEtlConfiguration()
{
ConnectionStringName = csName,
Transforms =
{
new Transformation()
{
Name = "loadAll",
Collections ={ "Users" },
Script = "loadToUsers(this)"
}
}
};
await store.Maintenance.SendAsync(new AddEtlOperation <RavenConnectionString>(etlConfiguration));
await store.Maintenance.SendAsync(new AddEtlOperation <RavenConnectionString>(etlConfiguration));
}
}
public void Should_throw_when_transforms_is_empty()
{
var configuration = new RavenEtlConfiguration {
ConnectionStringName = "test", Name = "myConfig", MentorNode = "A", Transforms = new List <Transformation>()
};
configuration.Initialize(new RavenConnectionString());
var e = Assert.Throws <InvalidOperationException>(() => configuration.Validate(out List <string> _));
Assert.Equal($"'{nameof(RavenEtlConfiguration.Transforms)}' list cannot be empty.", e.Message);
}
public void GetRavenEtlTaskInfo()
{
var etlConfiguration = new RavenEtlConfiguration()
{
Name = "test",
ConnectionStringName = "cs",
Transforms =
{
new Transformation()
{
Name = "loadAll",
Collections ={ "Users" },
Script = "loadToUsers(this)"
}
}
};
using (var store = GetDocumentStore())
{
var result = store.Maintenance.Send(new PutConnectionStringOperation <RavenConnectionString>(new RavenConnectionString
{
Name = "cs",
TopologyDiscoveryUrls = new[] { "http://127.0.0.1:8080" },
Database = "Northwind",
}));
Assert.NotNull(result.RaftCommandIndex);
var ravenEtlResult = store.Maintenance.Send(new AddEtlOperation <RavenConnectionString>(etlConfiguration));
var taskId = ravenEtlResult.TaskId;
var op = new GetOngoingTaskInfoOperation(taskId, OngoingTaskType.RavenEtl);
var etlResult = (OngoingTaskRavenEtlDetails)store.Maintenance.Send(op);
Assert.Equal("cs", etlResult.Configuration.ConnectionStringName);
Assert.Equal("test", etlResult.Configuration.Name);
Assert.Equal("loadAll", etlResult.Configuration.Transforms[0].Name);
Assert.Equal("loadToUsers(this)", etlResult.Configuration.Transforms[0].Script);
Assert.Equal("Users", etlResult.Configuration.Transforms[0].Collections[0]);
Assert.Equal(etlConfiguration.Name, etlResult?.TaskName);
op = new GetOngoingTaskInfoOperation("test", OngoingTaskType.RavenEtl);
etlResult = (OngoingTaskRavenEtlDetails)store.Maintenance.Send(op);
Assert.Equal("cs", etlResult.Configuration.ConnectionStringName);
Assert.Equal(taskId, etlResult.TaskId);
Assert.Equal("loadAll", etlResult.Configuration.Transforms[0].Name);
Assert.Equal("loadToUsers(this)", etlResult.Configuration.Transforms[0].Script);
Assert.Equal("Users", etlResult.Configuration.Transforms[0].Collections[0]);
Assert.Equal(etlConfiguration.Name, etlResult?.TaskName);
}
}
public void EtlTaskDeletionShouldDeleteItsState()
{
using (var src = GetDocumentStore())
using (var dest = GetDocumentStore())
{
using (var session = src.OpenSession())
{
session.Store(new User());
session.SaveChanges();
}
var etlDone = WaitForEtl(src, (n, s) => s.LoadSuccesses > 0);
var configuration = new RavenEtlConfiguration()
{
ConnectionStringName = "test",
Name = "myConfiguration",
Transforms =
{
new Transformation()
{
Name = "allUsers",
Collections ={ "Users" }
}
}
};
var result = AddEtl(src, configuration, new RavenConnectionString()
{
Name = "test",
TopologyDiscoveryUrls = dest.Urls,
Database = dest.Database,
});
Assert.True(etlDone.Wait(TimeSpan.FromMinutes(1)));
src.Maintenance.Send(new DeleteOngoingTaskOperation(result.TaskId, OngoingTaskType.RavenEtl));
etlDone.Reset();
AddEtl(src, configuration, new RavenConnectionString()
{
Name = "test",
TopologyDiscoveryUrls = dest.Urls,
Database = dest.Database,
});
Assert.True(etlDone.Wait(TimeSpan.FromMinutes(1)));
}
}
public async Task CanGetBatchStopReasonFromEtlPerformanceStats()
{
using (var src = GetDocumentStore())
using (var dst = GetDocumentStore())
{
var configuration = new RavenEtlConfiguration
{
ConnectionStringName = "test",
Name = "aaa",
Transforms =
{
new Transformation
{
Name = "S1",
Collections ={ "Users" }
}
}
};
AddEtl(src, configuration, new RavenConnectionString
{
Name = "test",
TopologyDiscoveryUrls = dst.Urls,
Database = dst.Database,
});
var etlDone = WaitForEtl(src, (_, statistics) => statistics.LoadSuccesses == 10);
using (var session = src.OpenAsyncSession())
{
for (int i = 0; i < 10; i++)
{
await session.StoreAsync(new User());
}
await session.SaveChangesAsync();
}
Assert.True(etlDone.Wait(TimeSpan.FromSeconds(10)));
var database = await Databases.GetDocumentDatabaseInstanceFor(src);
var etlProcess = database.EtlLoader.Processes.First();
var performance = etlProcess.GetPerformanceStats();
Assert.Contains("Successfully finished loading all batch items", performance.Select(p => p.BatchStopReason));
Assert.Contains("No more items to process", performance.Select(p => p.BatchTransformationCompleteReason));
}
}
public void ShouldThrowOnInvalidConfigOnUpdate()
{
using (var src = GetDocumentStore())
using (var dest = GetDocumentStore())
{
using (var session = src.OpenSession())
{
session.Store(new Doc {
Id = "doc-1", StrVal = "doc-1", StrVal2 = "doc-1"
});
session.SaveChanges();
}
var putResult = src.Maintenance.Send(new PutConnectionStringOperation <RavenConnectionString>(new RavenConnectionString
{
Name = "test", TopologyDiscoveryUrls = dest.Urls, Database = dest.Database,
}));
var configuration = new RavenEtlConfiguration()
{
ConnectionStringName = "test",
Name = "myConfiguration",
Transforms = { new Transformation()
{
Name = "allDocs", Collections ={ "Docs" }, Script = @"loadToDocs({ StrVal: this.StrVal });",
} }
};
var addResult = src.Maintenance.Send(new AddEtlOperation <RavenConnectionString>(configuration));
var configuration2 = new RavenEtlConfiguration()
{
ConnectionStringName = "test",
Name = "myConfiguration",
Transforms =
{
new Transformation()
{
Name = "allDocs", Collections ={ "Docs" }, Script = @"loadToDocs({ StrVal: this.StrVal, StrVal2: this.StrVal2 });", ApplyToAllDocuments = true,
}
}
};
var ex = Assert.Throws <RavenException>(() => src.Maintenance.Send(new UpdateEtlOperation <RavenConnectionString>(addResult.TaskId, configuration2)));
Assert.Contains("Collections cannot be specified when ApplyToAllDocuments is set. Script name: 'allDocs'", ex.Message);
}
}
public void EntersFallbackModeIfCantConnectTheDestination()
{
using (var src = GetDocumentStore())
{
using (var store = src.OpenSession())
{
store.Store(new User());
store.SaveChanges();
}
var configuration = new RavenEtlConfiguration()
{
ConnectionStringName = "test",
Name = "aaa",
Transforms =
{
new Transformation()
{
Collections ={ "Users" },
Name = "test"
}
}
};
AddEtl(src, configuration, new RavenConnectionString
{
Name = "test",
TopologyDiscoveryUrls = new [] { "http://abc.localhost:1234" },
Database = "test",
});
var process = GetDatabase(src.Database).Result.EtlLoader.Processes[0];
Assert.True(SpinWait.SpinUntil(() =>
{
if (process.FallbackTime != null)
{
return(true);
}
Thread.Sleep(100);
return(false);
}, TimeSpan.FromMinutes(1)));
}
}
public void CanResetEtl()
{
using (var src = GetDocumentStore())
using (var dest = GetDocumentStore())
{
using (var session = src.OpenSession())
{
session.Store(new User());
session.SaveChanges();
}
var runs = 0;
var etlDone = WaitForEtl(src, (n, s) => s.LoadSuccesses > 0);
var resetDone = WaitForEtl(src, (n, statistics) => ++ runs >= 2);
var configuration = new RavenEtlConfiguration()
{
ConnectionStringName = "test",
Name = "myConfiguration",
Transforms =
{
new Transformation()
{
Name = "allUsers",
Collections ={ "Users" }
}
}
};
AddEtl(src, configuration, new RavenConnectionString()
{
Name = "test",
TopologyDiscoveryUrls = dest.Urls,
Database = dest.Database,
});
Assert.True(etlDone.Wait(TimeSpan.FromMinutes(1)));
src.Admin.Server.Send(new ResetEtlOperation("myConfiguration", "allUsers", src.Database));
Assert.True(resetDone.Wait(TimeSpan.FromMinutes(1)));
}
}
public async Task CanResetEtl3()
{
using (var src = GetDocumentStore())
using (var dest = GetDocumentStore())
{
var configuration = new RavenEtlConfiguration()
{
ConnectionStringName = "test",
Name = "myConfiguration",
Transforms = { new Transformation()
{
Name = "allUsers", Collections ={ "Users" }
} }
};
AddEtl(src, configuration, new RavenConnectionString {
Name = "test", TopologyDiscoveryUrls = dest.Urls, Database = dest.Database,
});
var t = Task.Run(async() =>
{
for (int i = 0; i < 100; i++)
{
await src.Maintenance.SendAsync(new ResetEtlOperation("myConfiguration", "allUsers"));
await Task.Delay(100);
}
});
var indexes = new List <Task>();
for (int i = 0; i < 100; i++)
{
var index = new Index($"test{i}");
indexes.Add(index.ExecuteAsync(src));
}
await Task.WhenAll(indexes);
await t;
var record = await src.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(src.Database));
Assert.Equal(100, record.Indexes.Count);
}
}
public async Task CanUpdateEtl()
{
using (var store = GetDocumentStore())
{
var database = await GetDatabase(store.Database);
var configuration = new RavenEtlConfiguration()
{
ConnectionStringName = "test",
Name = "aaa",
Transforms =
{
new Transformation()
{
Collections ={ "Users" }
},
new Transformation()
{
Collections ={ "Users" }
}
}
};
var result = AddEtl(store, configuration, new RavenConnectionString()
{
Name = "test",
TopologyDiscoveryUrls = new[] { "http://127.0.0.1:8080" },
Database = "Northwind",
});
configuration.Transforms[0].Disabled = true;
var update = store.Admin.Server.Send(new UpdateEtlOperation <RavenConnectionString>(result.TaskId, configuration, database.Name));
var ongoingTask = store.Admin.Server.Send(new GetOngoingTaskInfoOperation(store.Database, update.TaskId, OngoingTaskType.RavenEtl));
Assert.Equal(OngoingTaskState.PartiallyEnabled, ongoingTask.TaskState);
}
}
public void Error_if_script_does_not_contain_any_loadTo_method_and_isnt_empty()
{
var config = new RavenEtlConfiguration
{
Name = "test",
ConnectionStringName = "test",
Transforms =
{
new Transformation
{
Name = "test",
Collections ={ "Users" },
Script = @"
this.Name = 'aaa'; // we have some modification but we don't call loadTo method so it should error
function deleteDocumentsOfUsersBehavior(docId) {
return false;
}
"
}
}
};
config.Initialize(new RavenConnectionString()
{
Database = "Foo", TopologyDiscoveryUrls = new[] { "http://localhost:8080" }
});
List <string> errors;
config.Validate(out errors);
Assert.Equal(1, errors.Count);
Assert.Equal("No `loadTo<CollectionName>()` method call found in 'test' script", errors[0]);
}
private IEnumerable <EtlProcess> GetRelevantProcesses <T, TConnectionString>(List <T> configurations, HashSet <string> uniqueNames) where T : EtlConfiguration <TConnectionString> where TConnectionString : ConnectionString
{
foreach (var config in configurations)
{
SqlEtlConfiguration sqlConfig = null;
RavenEtlConfiguration ravenConfig = null;
var connectionStringNotFound = false;
switch (config.EtlType)
{
case EtlType.Raven:
ravenConfig = config as RavenEtlConfiguration;
if (_databaseRecord.RavenConnectionStrings.TryGetValue(config.ConnectionStringName, out var ravenConnection))
{
ravenConfig.Initialize(ravenConnection);
}
else
{
connectionStringNotFound = true;
}
break;
case EtlType.Sql:
sqlConfig = config as SqlEtlConfiguration;
if (_databaseRecord.SqlConnectionStrings.TryGetValue(config.ConnectionStringName, out var sqlConnection))
{
sqlConfig.Initialize(sqlConnection);
}
else
{
connectionStringNotFound = true;
}
break;
default:
ThrownUnknownEtlConfiguration(config.GetType());
break;
}
if (connectionStringNotFound)
{
LogConfigurationError(config,
new List <string>
{
$"Connection string named '{config.ConnectionStringName}' was not found for {config.EtlType} ETL"
});
continue;
}
if (config.Disabled)
{
continue;
}
if (ValidateConfiguration(config, uniqueNames) == false)
{
continue;
}
var processState = GetProcessState(config.Transforms, _database, config.Name);
var whoseTaskIsIt = _database.WhoseTaskIsIt(_databaseRecord.Topology, config, processState);
if (whoseTaskIsIt != _serverStore.NodeTag)
{
continue;
}
foreach (var transform in config.Transforms)
{
if (transform.Disabled)
{
continue;
}
EtlProcess process = null;
if (sqlConfig != null)
{
process = new SqlEtl(transform, sqlConfig, _database, _serverStore);
}
if (ravenConfig != null)
{
process = new RavenEtl(transform, ravenConfig, _database, _serverStore);
}
yield return(process);
}
}
}
public async Task Can_fail_over_etl_task()
{
var(nodes, leader) = await CreateRaftCluster(3, customSettings :
new Dictionary <string, string>
{
{ "ETL.MaxNumberOfExtractedDocuments", "5" }
});
var database = GetDatabaseName();
await CreateDatabaseInClusterInner(new DatabaseRecord(database), 3, leader.WebUrl, null);
using (var destination = GetDocumentStore())
using (var source = new DocumentStore
{
Database = database,
Urls = new[] { leader.WebUrl }
}.Initialize())
{
string lastDocumentId = null;
using (var bulkInsert = source.BulkInsert())
{
for (var i = 0; i < 10; i++)
{
lastDocumentId = i.ToString();
await bulkInsert.StoreAsync(new User(), lastDocumentId);
}
}
var putResult = await source.Maintenance.SendAsync(new PutConnectionStringOperation <RavenConnectionString>(new RavenConnectionString
{
Name = "test",
TopologyDiscoveryUrls = new[] { destination.Urls.First() },
Database = destination.Database,
}));
Assert.NotNull(putResult.RaftCommandIndex);
var configuration = new RavenEtlConfiguration
{
ConnectionStringName = "test",
Name = "myConfiguration",
Transforms =
{
new Transformation
{
Name = "allDocs",
Collections ={ "Users" }
}
},
MentorNode = "A"
};
var addResult = await source.Maintenance.SendAsync(new AddEtlOperation <RavenConnectionString>(configuration));
Assert.NotNull(addResult.RaftCommandIndex);
Assert.True(WaitForDocument(destination, lastDocumentId));
configuration.MentorNode = "B";
var updateResult = await source.Maintenance.SendAsync(new UpdateEtlOperation <RavenConnectionString>(addResult.TaskId, configuration));
Assert.NotNull(updateResult.RaftCommandIndex);
using (var session = source.OpenAsyncSession())
{
var user = new User();
await session.StoreAsync(user);
lastDocumentId = user.Id;
await session.SaveChangesAsync();
}
Assert.True(WaitForDocument(destination, lastDocumentId));
}
}
public async Task ReplicateFromSingleSource()
{
var srcDb = "ReplicateFromSingleSourceSrc";
var dstDb = "ReplicateFromSingleSourceDst";
var(_, srcRaft) = await CreateRaftCluster(3);
var(_, dstRaft) = await CreateRaftCluster(1);
var srcNodes = await CreateDatabaseInCluster(srcDb, 3, srcRaft.WebUrl);
var destNode = await CreateDatabaseInCluster(dstDb, 1, dstRaft.WebUrl);
var node = srcNodes.Servers.First(x => x.ServerStore.NodeTag != srcRaft.ServerStore.NodeTag).ServerStore.NodeTag;
using (var src = new DocumentStore
{
Urls = srcNodes.Servers.Select(s => s.WebUrl).ToArray(),
Database = srcDb,
}.Initialize())
using (var dest = new DocumentStore
{
Urls = new[] { destNode.Servers[0].WebUrl },
Database = dstDb,
}.Initialize())
{
var connectionStringName = "EtlFailover";
var urls = new[] { destNode.Servers[0].WebUrl };
var config = new RavenEtlConfiguration()
{
Name = connectionStringName,
ConnectionStringName = connectionStringName,
Transforms =
{
new Transformation
{
Name = $"ETL : {connectionStringName}",
Collections = new List <string>(new[] { "Users" }),
Script = null,
ApplyToAllDocuments = false,
Disabled = false
}
},
LoadRequestTimeoutInSec = 30,
MentorNode = node
};
var connectionString = new RavenConnectionString
{
Name = connectionStringName,
Database = dest.Database,
TopologyDiscoveryUrls = urls,
};
var result = src.Maintenance.Send(new PutConnectionStringOperation <RavenConnectionString>(connectionString));
Assert.NotNull(result.RaftCommandIndex);
src.Maintenance.Send(new AddEtlOperation <RavenConnectionString>(config));
var originalTaskNode = srcNodes.Servers.Single(s => s.ServerStore.NodeTag == node);
using (var session = src.OpenSession())
{
session.Store(new User()
{
Name = "Joe Doe"
}, "users/1");
session.SaveChanges();
}
Assert.True(WaitForDocument <User>(dest, "users/1", u => u.Name == "Joe Doe", 30_000));
await ActionWithLeader((l) => l.ServerStore.RemoveFromClusterAsync(node));
await originalTaskNode.ServerStore.WaitForState(RachisState.Passive, CancellationToken.None);
using (var session = src.OpenSession())
{
session.Store(new User()
{
Name = "Joe Doe2"
}, "users/2");
session.SaveChanges();
}
Assert.True(WaitForDocument <User>(dest, "users/2", u => u.Name == "Joe Doe2", 30_000));
Assert.Throws <NodeIsPassiveException>(() =>
{
using (var originalSrc = new DocumentStore
{
Urls = new[] { originalTaskNode.WebUrl },
Database = srcDb,
Conventions = new DocumentConventions
{
DisableTopologyUpdates = true
}
}.Initialize())
{
using (var session = originalSrc.OpenSession())
{
session.Store(new User()
{
Name = "Joe Doe3"
}, "users/3");
session.SaveChanges();
}
}
});
}
}
public async Task AutoNamingAlgorithmOfOngoingTasksShouldTakeNameAlreadyExistsIntoAccount()
{
using (var store = GetDocumentStore())
{
var dbName = $"db/{Guid.NewGuid()}";
var csName = $"cs/{Guid.NewGuid()}";
var connectionString = new RavenConnectionString
{
Name = csName,
Database = dbName,
TopologyDiscoveryUrls = new[] { "http://127.0.0.1:12345" }
};
var result = await store.Maintenance.SendAsync(new PutConnectionStringOperation <RavenConnectionString>(connectionString));
Assert.NotNull(result.RaftCommandIndex);
await store.Maintenance.SendAsync(new UpdateExternalReplicationOperation(new ExternalReplication(dbName, csName)));
await store.Maintenance.SendAsync(new UpdateExternalReplicationOperation(new ExternalReplication(dbName, csName)));
var backupConfig = Backup.CreateBackupConfiguration(backupPath: NewDataPath(suffix: "BackupFolder"), fullBackupFrequency: "* */1 * * *", incrementalBackupFrequency: "* */2 * * *", azureSettings: new AzureSettings
{
StorageContainer = "abc"
}, disabled: true);
await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfig));
await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfig));
var etlConfiguration = new RavenEtlConfiguration
{
ConnectionStringName = csName,
Transforms =
{
new Transformation()
{
Name = "loadAll",
Collections ={ "Users" },
Script = "loadToUsers(this)"
}
}
};
await store.Maintenance.SendAsync(new AddEtlOperation <RavenConnectionString>(etlConfiguration));
await store.Maintenance.SendAsync(new AddEtlOperation <RavenConnectionString>(etlConfiguration));
// for Pull Replication Hub name is required - no need to test
var sink = new PullReplicationAsSink
{
HubDefinitionName = "aa",
ConnectionString = connectionString,
ConnectionStringName = connectionString.Name
};
await store.Maintenance.SendAsync(new UpdatePullReplicationAsSinkOperation(sink));
await store.Maintenance.SendAsync(new UpdatePullReplicationAsSinkOperation(sink));
}
}
public async Task ShouldSendCounterChangeMadeInCluster()
{
var srcDb = "13288-src";
var dstDb = "13288-dst";
var(_, srcRaft) = await CreateRaftCluster(2);
var(_, dstRaft) = await CreateRaftCluster(1);
var srcNodes = await CreateDatabaseInCluster(srcDb, 2, srcRaft.WebUrl);
var destNode = await CreateDatabaseInCluster(dstDb, 1, dstRaft.WebUrl);
using (var src = new DocumentStore
{
Urls = srcNodes.Servers.Select(s => s.WebUrl).ToArray(),
Database = srcDb,
}.Initialize())
using (var dest = new DocumentStore
{
Urls = new[] { destNode.Servers[0].WebUrl },
Database = dstDb,
}.Initialize())
{
var connectionStringName = "my-etl";
var urls = new[] { destNode.Servers[0].WebUrl };
var config = new RavenEtlConfiguration()
{
Name = connectionStringName,
ConnectionStringName = connectionStringName,
Transforms =
{
new Transformation
{
Name = $"ETL : {connectionStringName}",
Collections = new List <string>(new[] { "Users" }),
Script = null,
ApplyToAllDocuments = false,
Disabled = false
}
},
LoadRequestTimeoutInSec = 30,
MentorNode = "A"
};
var connectionString = new RavenConnectionString
{
Name = connectionStringName,
Database = dest.Database,
TopologyDiscoveryUrls = urls,
};
var result = src.Maintenance.Send(new PutConnectionStringOperation <RavenConnectionString>(connectionString));
Assert.NotNull(result.RaftCommandIndex);
src.Maintenance.Send(new AddEtlOperation <RavenConnectionString>(config));
var aNode = srcNodes.Servers.Single(s => s.ServerStore.NodeTag == "A");
var bNode = srcNodes.Servers.Single(s => s.ServerStore.NodeTag == "B");
// modify counter on A node (mentor of ETL task)
using (var aSrc = new DocumentStore
{
Urls = new[] { aNode.WebUrl },
Database = srcDb,
Conventions = new DocumentConventions
{
DisableTopologyUpdates = true
}
}.Initialize())
{
using (var session = aSrc.OpenSession())
{
session.Store(new User()
{
Name = "Joe Doe"
}, "users/1");
session.CountersFor("users/1").Increment("likes");
session.Advanced.WaitForReplicationAfterSaveChanges();
session.SaveChanges();
}
}
Assert.True(WaitForDocument <User>(dest, "users/1", u => u.Name == "Joe Doe", 30_000));
using (var session = dest.OpenSession())
{
var user = session.Load <User>("users/1");
Assert.NotNull(user);
Assert.Equal("Joe Doe", user.Name);
var counter = session.CountersFor("users/1").Get("likes");
Assert.NotNull(counter);
Assert.Equal(1, counter.Value);
}
// modify counter on B node (not mentor)
using (var bSrc = new DocumentStore
{
Urls = new[] { bNode.WebUrl },
Database = srcDb,
Conventions = new DocumentConventions
{
DisableTopologyUpdates = true
}
}.Initialize())
{
using (var session = bSrc.OpenSession())
{
session.CountersFor("users/1").Increment("likes");
session.SaveChanges();
}
}
Assert.True(Replication.WaitForCounterReplication(new List <IDocumentStore>
{
dest
}, "users/1", "likes", 2, TimeSpan.FromSeconds(60)));
}
}
public void HandleDatabaseRecordChange(DatabaseRecord record)
{
if (record == null)
{
return;
}
var myRavenEtl = new List <RavenEtlConfiguration>();
var mySqlEtl = new List <SqlEtlConfiguration>();
foreach (var config in record.RavenEtls)
{
if (IsMyEtlTask <RavenEtlConfiguration, RavenConnectionString>(record, config))
{
myRavenEtl.Add(config);
}
}
foreach (var config in record.SqlEtls)
{
if (IsMyEtlTask <SqlEtlConfiguration, SqlConnectionString>(record, config))
{
mySqlEtl.Add(config);
}
}
var toRemove = _processes.GroupBy(x => x.ConfigurationName).ToDictionary(x => x.Key, x => x.ToList());
foreach (var processesPerConfig in _processes.GroupBy(x => x.ConfigurationName))
{
var process = processesPerConfig.First();
Debug.Assert(processesPerConfig.All(x => x.GetType() == process.GetType()));
if (process is RavenEtl ravenEtl)
{
RavenEtlConfiguration existing = null;
foreach (var config in myRavenEtl)
{
if (ravenEtl.Configuration.IsEqual(config))
{
existing = config;
break;
}
}
if (existing != null)
{
toRemove.Remove(processesPerConfig.Key);
myRavenEtl.Remove(existing);
}
}
else if (process is SqlEtl sqlEtl)
{
SqlEtlConfiguration existing = null;
foreach (var config in mySqlEtl)
{
if (sqlEtl.Configuration.IsEqual(config))
{
existing = config;
break;
}
}
if (existing != null)
{
toRemove.Remove(processesPerConfig.Key);
mySqlEtl.Remove(existing);
}
}
else
{
throw new InvalidOperationException($"Unknown ETL process type: {process.GetType()}");
}
}
Parallel.ForEach(toRemove, x =>
{
foreach (var process in x.Value)
{
try
{
process.Stop();
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info($"Failed to stop ETL process {process.Name} on the database record change", e);
}
}
}
});
LoadProcesses(record, myRavenEtl, mySqlEtl, toRemove.SelectMany(x => x.Value).ToList());
// unsubscribe old etls _after_ we start new processes to ensure the tombstone cleaner
// constantly keeps track of tombstones processed by ETLs so it won't delete them during etl processes reloading
foreach (var processesPerConfig in toRemove)
{
foreach (var process in processesPerConfig.Value)
{
_database.TombstoneCleaner.Unsubscribe(process);
}
}
Parallel.ForEach(toRemove, x =>
{
foreach (var process in x.Value)
{
try
{
process.Dispose();
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info($"Failed to dispose ETL process {process.Name} on the database record change", e);
}
}
}
});
}
public async Task CanGetTaskInfo()
{
var clusterSize = 3;
var databaseName = "TestDB";
var leader = await CreateRaftClusterAndGetLeader(clusterSize);
ModifyOngoingTaskResult addWatcherRes;
UpdatePeriodicBackupOperationResult updateBackupResult;
AddEtlOperationResult addRavenEtlResult;
AddEtlOperationResult addSqlEtlResult;
RavenEtlConfiguration etlConfiguration;
SqlEtlConfiguration sqlConfiguration;
ExternalReplication watcher;
SqlConnectionString sqlConnectionString;
var sqlScript = @"
var orderData = {
Id: __document_id,
OrderLinesCount: this.OrderLines.length,
TotalCost: 0
};
loadToOrders(orderData);
";
using (var store = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName
}.Initialize())
{
var doc = new DatabaseRecord(databaseName);
var databaseResult = await store.Admin.Server.SendAsync(new CreateDatabaseOperation(doc, clusterSize));
Assert.Equal(clusterSize, databaseResult.Topology.AllNodes.Count());
foreach (var server in Servers)
{
await server.ServerStore.Cluster.WaitForIndexNotification(databaseResult.RaftCommandIndex);
}
foreach (var server in Servers)
{
await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName);
}
watcher = new ExternalReplication
{
Database = "Watcher1",
Url = "http://127.0.0.1:9090",
Name = "MyExternalReplication"
};
addWatcherRes = await AddWatcherToReplicationTopology((DocumentStore)store, watcher);
var backupConfig = new PeriodicBackupConfiguration
{
Name = "backup1",
LocalSettings = new LocalSettings
{
FolderPath = NewDataPath(suffix: "BackupFolder")
},
AzureSettings = new AzureSettings
{
StorageContainer = "abc"
},
FullBackupFrequency = "* */1 * * *",
IncrementalBackupFrequency = "* */2 * * *",
Disabled = true
};
updateBackupResult = await store.Admin.Server.SendAsync(new UpdatePeriodicBackupOperation(backupConfig, store.Database));
store.Admin.Server.Send(new PutConnectionStringOperation <RavenConnectionString>(new RavenConnectionString
{
Name = "cs",
Url = "http://127.0.0.1:8080",
Database = "Northwind",
}, store.Database));
etlConfiguration = new RavenEtlConfiguration()
{
Name = "tesst",
ConnectionStringName = "cs",
Transforms =
{
new Transformation()
{
Name = "loadAll",
Collections ={ "Users" },
Script = "loadToUsers(this)"
}
}
};
addRavenEtlResult = store.Admin.Server.Send(new AddEtlOperation <RavenConnectionString>(etlConfiguration, store.Database));
sqlConnectionString = new SqlConnectionString
{
Name = "abc",
ConnectionString = @"Data Source=localhost\sqlexpress;Integrated Security=SSPI;Connection Timeout=3" + $";Initial Catalog=SqlReplication-{store.Database};"
};
store.Admin.Server.Send(new PutConnectionStringOperation <SqlConnectionString>(sqlConnectionString, store.Database));
sqlConfiguration = new SqlEtlConfiguration()
{
Name = "abc",
ConnectionStringName = "abc",
FactoryName = "System.Data.SqlClient",
SqlTables =
{
new SqlEtlTable {
TableName = "Orders", DocumentIdColumn = "Id", InsertOnlyMode = false
},
new SqlEtlTable {
TableName = "OrderLines", DocumentIdColumn = "OrderId", InsertOnlyMode = false
},
},
Transforms =
{
new Transformation()
{
Name = "OrdersAndLines",
Collections = new List <string>{
"Orders"
},
Script = sqlScript
}
}
};
addSqlEtlResult = store.Admin.Server.Send(new AddEtlOperation <SqlConnectionString>(sqlConfiguration, store.Database));
}
using (var store = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName,
Conventions =
{
DisableTopologyUpdates = true
}
}.Initialize())
{
var taskId = addWatcherRes.TaskId;
var replicationResult = (OngoingTaskReplication) await GetTaskInfo((DocumentStore)store, taskId, OngoingTaskType.Replication);
Assert.Equal(watcher.Database, replicationResult.DestinationDatabase);
Assert.Equal(watcher.Url, replicationResult.DestinationUrl);
Assert.Equal(watcher.Name, replicationResult.TaskName);
taskId = updateBackupResult.TaskId;
var backupResult = (OngoingTaskBackup) await GetTaskInfo((DocumentStore)store, taskId, OngoingTaskType.Backup);
Assert.Equal("Local", backupResult.BackupDestinations[0]);
Assert.Equal("Azure", backupResult.BackupDestinations[1]);
Assert.Equal("backup1", backupResult.TaskName);
Assert.Equal(OngoingTaskState.Disabled, backupResult.TaskState);
taskId = addRavenEtlResult.TaskId;
var etlResult = (OngoingTaskRavenEtl) await GetTaskInfo((DocumentStore)store, taskId, OngoingTaskType.RavenEtl);
Assert.Equal("cs", etlResult.Configuration.ConnectionStringName);
Assert.Equal("tesst", etlResult.Configuration.Name);
Assert.Equal("loadAll", etlResult.Configuration.Transforms[0].Name);
Assert.Equal("loadToUsers(this)", etlResult.Configuration.Transforms[0].Script);
Assert.Equal("Users", etlResult.Configuration.Transforms[0].Collections[0]);
Assert.Equal(etlConfiguration.Name, etlResult?.TaskName);
taskId = addSqlEtlResult.TaskId;
var sqlResult = (OngoingTaskSqlEtl) await GetTaskInfo((DocumentStore)store, taskId, OngoingTaskType.SqlEtl);
Assert.Equal("abc", sqlResult.Configuration.ConnectionStringName);
Assert.Equal("abc", sqlResult.Configuration.Name);
Assert.Equal("OrdersAndLines", sqlResult.Configuration.Transforms[0].Name);
Assert.Equal(sqlScript, sqlResult.Configuration.Transforms[0].Script);
Assert.Equal("Orders", sqlResult.Configuration.Transforms[0].Collections[0]);
Assert.NotNull(sqlResult.Configuration.SqlTables);
Assert.Equal(sqlConfiguration.Name, sqlResult?.TaskName);
}
}
public async Task WillWorkAfterResponsibleNodeRestart_RavenDB_13237()
{
var srcDb = "ETL-src";
var dstDb = "ETL-dst";
var(_, srcRaft) = await CreateRaftCluster(3, shouldRunInMemory : false);
var(_, dstRaft) = await CreateRaftCluster(1);
var srcNodes = await CreateDatabaseInCluster(srcDb, 2, srcRaft.WebUrl);
var destNode = await CreateDatabaseInCluster(dstDb, 1, dstRaft.WebUrl);
using (var src = new DocumentStore
{
Urls = srcNodes.Servers.Select(s => s.WebUrl).ToArray(),
Database = srcDb,
}.Initialize())
using (var dest = new DocumentStore
{
Urls = new[] { destNode.Servers[0].WebUrl },
Database = dstDb,
}.Initialize())
{
var name = "FailoverAfterRestart";
var urls = new[] { destNode.Servers[0].WebUrl };
var config = new RavenEtlConfiguration()
{
Name = name,
ConnectionStringName = name,
Transforms =
{
new Transformation
{
Name = $"ETL : {name}",
Collections = new List <string>(new[] { "Users" }),
Script = null,
ApplyToAllDocuments = false,
Disabled = false
}
},
LoadRequestTimeoutInSec = 30,
};
var connectionString = new RavenConnectionString
{
Name = name,
Database = dest.Database,
TopologyDiscoveryUrls = urls,
};
var result = src.Maintenance.Send(new PutConnectionStringOperation <RavenConnectionString>(connectionString));
Assert.NotNull(result.RaftCommandIndex);
src.Maintenance.Send(new AddEtlOperation <RavenConnectionString>(config));
var ongoingTask = src.Maintenance.Send(new GetOngoingTaskInfoOperation(name, OngoingTaskType.RavenEtl));
var responsibleNodeNodeTag = ongoingTask.ResponsibleNode.NodeTag;
var originalTaskNodeServer = srcNodes.Servers.Single(s => s.ServerStore.NodeTag == responsibleNodeNodeTag);
using (var session = src.OpenSession())
{
session.Store(new User()
{
Name = "Joe Doe"
}, "users/1");
session.SaveChanges();
}
Assert.True(WaitForDocument <User>(dest, "users/1", u => u.Name == "Joe Doe", 30_000));
var originalResult = DisposeServerAndWaitForFinishOfDisposal(originalTaskNodeServer);
using (var session = src.OpenSession())
{
session.Store(new User()
{
Name = "Joe Doe2"
}, "users/2");
session.SaveChanges();
}
Assert.True(WaitForDocument <User>(dest, "users/2", u => u.Name == "Joe Doe2", 30_000));
ongoingTask = src.Maintenance.Send(new GetOngoingTaskInfoOperation(name, OngoingTaskType.RavenEtl));
var currentNodeNodeTag = ongoingTask.ResponsibleNode.NodeTag;
var currentTaskNodeServer = srcNodes.Servers.Single(s => s.ServerStore.NodeTag == currentNodeNodeTag);
// start server which originally was handling ETL task
GetNewServer(new ServerCreationOptions
{
CustomSettings = new Dictionary <string, string>
{
{ RavenConfiguration.GetKey(x => x.Core.ServerUrls), originalResult.Url }
},
RunInMemory = false,
DeletePrevious = false,
DataDirectory = originalResult.DataDirectory
});
using (var store = new DocumentStore
{
Urls = new[] { originalResult.Url },
Database = srcDb,
Conventions =
{
DisableTopologyUpdates = true
}
}.Initialize())
{
using (var session = store.OpenSession())
{
session.Store(new User()
{
Name = "Joe Doe3"
}, "users/3");
session.SaveChanges();
}
Assert.True(WaitForDocument <User>(dest, "users/3", u => u.Name == "Joe Doe3", 30_000));
// force disposing second node to ensure the original node is reponsible for ETL task again
DisposeServerAndWaitForFinishOfDisposal(currentTaskNodeServer);
using (var session = store.OpenSession())
{
session.Store(new User()
{
Name = "Joe Doe4"
}, "users/4");
session.SaveChanges();
}
Assert.True(WaitForDocument <User>(dest, "users/4", u => u.Name == "Joe Doe4", 30_000));
}
}
}
public async Task EtlTombstonesInTheCluster()
{
var cluster = await CreateRaftCluster(3);
using (var store = GetDocumentStore(new Options
{
Server = cluster.Leader,
ModifyDocumentStore = s => s.Conventions = new DocumentConventions(),
ReplicationFactor = 3
}))
using (var dest = GetDocumentStore())
{
var connectionString = await store.Maintenance.SendAsync(new PutConnectionStringOperation <RavenConnectionString>(new RavenConnectionString
{
Name = "test",
TopologyDiscoveryUrls = dest.Urls,
Database = dest.Database,
}));
Assert.NotNull(connectionString.RaftCommandIndex);
var configuration = new RavenEtlConfiguration()
{
ConnectionStringName = "test",
Name = "myConfiguration",
MentorNode = "A",
Transforms = new List <Transformation>
{
new Transformation
{
ApplyToAllDocuments = true,
Name = "blah"
}
}
};
await store.Maintenance.SendAsync(new AddEtlOperation <RavenConnectionString>(configuration));
var etlStorage = await cluster.Nodes.Single(n => n.ServerStore.NodeTag == "A").ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
var mre = new ManualResetEventSlim(false);
var sent = new ManualResetEventSlim(false);
etlStorage.EtlLoader.BatchCompleted += _ =>
{
sent.Set();
mre.Wait(TimeSpan.FromSeconds(30));
};
using (var session = store.OpenSession())
{
session.Advanced.WaitForReplicationAfterSaveChanges(replicas: 2);
session.Store(new User {
Name = "Karmel"
}, "foo/bar");
session.SaveChanges();
}
if (sent.Wait(TimeSpan.FromSeconds(30)) == false)
{
Assert.False(true, "timeout!");
}
using (var session = store.OpenSession())
{
session.Advanced.WaitForReplicationAfterSaveChanges(replicas: 2);
session.Delete("foo/bar");
session.SaveChanges();
}
using (var session = store.OpenSession())
{
session.Advanced.WaitForReplicationAfterSaveChanges(replicas: 2);
session.Store(new User {
Name = "Karmel"
}, "marker");
session.SaveChanges();
Assert.True(await WaitForDocumentInClusterAsync <User>((DocumentSession)session, "marker", (u) => u.Id == "marker", TimeSpan.FromSeconds(15)));
}
var total = 0L;
foreach (var server in cluster.Nodes)
{
var storage = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
using (storage.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
total += storage.DocumentsStorage.GetNumberOfTombstones(context);
}
}
Assert.Equal(3, total);
total = 0L;
foreach (var server in cluster.Nodes)
{
var storage = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
await storage.TombstoneCleaner.ExecuteCleanup();
using (storage.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
total += storage.DocumentsStorage.GetNumberOfTombstones(context);
}
}
Assert.Equal(3, total); // we didn't send the tombstone so we must not purge it
await WaitForLastReplicationEtag(cluster, store);
await DisposeServerAndWaitForFinishOfDisposalAsync(etlStorage.ServerStore.Server);
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "marker2");
session.SaveChanges();
}
WaitForDocument(dest, "marker2");
string changeVectorMarker2;
using (var session = dest.OpenSession())
{
Assert.Null(session.Load <User>("foo/bar"));
}
using (var session = store.OpenSession())
{
var marker = session.Load <User>("marker2");
changeVectorMarker2 = session.Advanced.GetChangeVectorFor(marker);
}
await ActionWithLeader((l) => WaitForRaftCommandToBeAppliedInCluster(l, nameof(UpdateEtlProcessStateCommand)));
Assert.True(await WaitForEtlState(cluster, store, changeVectorMarker2));
total = 0;
foreach (var server in cluster.Nodes)
{
if (server.Disposed)
{
continue;
}
var storage = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
await storage.TombstoneCleaner.ExecuteCleanup();
using (storage.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
total += storage.DocumentsStorage.GetNumberOfTombstones(context);
}
}
Assert.Equal(0, total);
}
}
public void HandleDatabaseRecordChange(DatabaseRecord record)
{
if (record == null)
{
return;
}
var myRavenEtl = new List <RavenEtlConfiguration>();
var mySqlEtl = new List <SqlEtlConfiguration>();
foreach (var config in record.RavenEtls)
{
if (IsMyEtlTask <RavenEtlConfiguration, RavenConnectionString>(record, config))
{
myRavenEtl.Add(config);
}
}
foreach (var config in record.SqlEtls)
{
if (IsMyEtlTask <SqlEtlConfiguration, SqlConnectionString>(record, config))
{
mySqlEtl.Add(config);
}
}
var toRemove = new List <EtlProcess>(_processes);
foreach (var etlProcess in _processes)
{
if (etlProcess is RavenEtl ravenEtl)
{
RavenEtlConfiguration existing = null;
foreach (var config in myRavenEtl)
{
if (ravenEtl.Configuration.IsEqual(config))
{
existing = config;
break;
}
}
if (existing != null)
{
toRemove.Remove(etlProcess);
myRavenEtl.Remove(existing);
}
}
if (etlProcess is SqlEtl sqlEtl)
{
SqlEtlConfiguration existing = null;
foreach (var config in mySqlEtl)
{
if (sqlEtl.Configuration.IsEqual(config))
{
existing = config;
break;
}
}
if (existing != null)
{
toRemove.Remove(etlProcess);
mySqlEtl.Remove(existing);
}
}
}
Parallel.ForEach(toRemove, x =>
{
try
{
x.Stop();
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info($"Failed to stop ETL process {x.Name} on the database record change", e);
}
}
});
LoadProcesses(record, myRavenEtl, mySqlEtl, toRemove);
// unsubscribe old etls _after_ we start new processes to ensure the tombstone cleaner
// constantly keeps track of tombstones processed by ETLs so it won't delete them during etl processes reloading
foreach (var process in toRemove)
{
_database.TombstoneCleaner.Unsubscribe(process);
}
Parallel.ForEach(toRemove, x =>
{
try
{
x.Dispose();
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info($"Failed to dispose ETL process {x.Name} on the database record change", e);
}
}
});
}
