public async Task MaintainWillCreateAliasOnVersionedIndex() {
var version1Index = new VersionedEmployeeIndex(_configuration, 1);
await version1Index.DeleteAsync();
var version2Index = new VersionedEmployeeIndex(_configuration, 2);
await version2Index.DeleteAsync();
// Indexes don't exist yet so the current version will be the index version.
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
Assert.Equal(2, await version2Index.GetCurrentVersionAsync());
using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
await version1Index.ConfigureAsync();
Assert.True(_client.IndexExists(version1Index.VersionedName).Exists);
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
await version2Index.ConfigureAsync();
Assert.True(_client.IndexExists(version2Index.VersionedName).Exists);
Assert.Equal(1, await version2Index.GetCurrentVersionAsync());
// delete all aliases
await _configuration.Cache.RemoveAllAsync();
await DeleteAliases(version1Index.VersionedName);
await DeleteAliases(version2Index.VersionedName);
await _client.RefreshAsync();
var aliasesResponse = await _client.GetAliasesAsync(a => a.Indices(version1Index.VersionedName, version2Index.VersionedName));
Assert.Equal(0, aliasesResponse.Indices.SelectMany(i => i.Value).Count());
// Indexes exist but no alias so the oldest index version will be used.
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
Assert.Equal(1, await version2Index.GetCurrentVersionAsync());
await version1Index.MaintainAsync();
aliasesResponse = await _client.GetAliasesAsync(a => a.Indices(version1Index.VersionedName));
Assert.Equal(1, aliasesResponse.Indices.Single().Value.Count);
aliasesResponse = await _client.GetAliasesAsync(a => a.Indices(version2Index.VersionedName));
Assert.Equal(0, aliasesResponse.Indices.Single().Value.Count);
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
Assert.Equal(1, await version2Index.GetCurrentVersionAsync());
}
}
}
public async Task CanReindexVersionedIndexWithDeletedDocs() {
var version1Index = new VersionedEmployeeIndex(_configuration, 1);
await version1Index.DeleteAsync();
var version2Index = new VersionedEmployeeIndex(_configuration, 2);
await version2Index.DeleteAsync();
using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
await version1Index.ConfigureAsync();
Assert.True(_client.IndexExists(version1Index.VersionedName).Exists);
var repository = new EmployeeRepository(version1Index.Employee);
var employee = await repository.AddAsync(EmployeeGenerator.Default);
Assert.NotNull(employee?.Id);
await _client.RefreshAsync();
using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
await version2Index.ConfigureAsync();
Assert.True(_client.IndexExists(version2Index.VersionedName).Exists);
Assert.Equal(1, await version2Index.GetCurrentVersionAsync());
// alias should still point to the old version until reindex
var aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Alias(version2Index.Name));
Assert.True(aliasResponse.IsValid);
Assert.Equal(1, aliasResponse.Indices.Count);
Assert.Equal(version1Index.VersionedName, aliasResponse.Indices.First().Key);
var countdown = new AsyncCountdownEvent(1);
var reindexTask = version2Index.ReindexAsync((progress, message) => {
_logger.Info($"Reindex Progress {progress}%: {message}");
if (progress == 95) {
countdown.Signal();
SystemClock.Sleep(1000);
}
return Task.CompletedTask;
});
// Wait until the first reindex pass is done.
await countdown.WaitAsync();
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
await repository.RemoveAllAsync();
await _client.RefreshAsync();
// Resume after everythings been indexed.
await reindexTask;
aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Alias(version2Index.Name));
Assert.True(aliasResponse.IsValid, aliasResponse.GetErrorMessage());
Assert.Equal(1, aliasResponse.Indices.Count);
Assert.Equal(version2Index.VersionedName, aliasResponse.Indices.First().Key);
Assert.Equal(2, await version1Index.GetCurrentVersionAsync());
Assert.Equal(2, await version2Index.GetCurrentVersionAsync());
var countResponse = await _client.CountAsync(d => d.Index(version1Index.VersionedName));
_logger.Trace(() => countResponse.GetRequest());
Assert.True(countResponse.ConnectionStatus.HttpStatusCode == 404, countResponse.GetErrorMessage());
Assert.Equal(0, countResponse.Count);
countResponse = await _client.CountAsync(d => d.Index(version2Index.VersionedName));
_logger.Trace(() => countResponse.GetRequest());
Assert.True(countResponse.IsValid, countResponse.GetErrorMessage());
Assert.Equal(1, countResponse.Count);
Assert.Equal(employee, await repository.GetByIdAsync(employee.Id));
Assert.False(_client.IndexExists(d => d.Index(version1Index.VersionedName)).Exists);
}
}
}
public async Task CanReindexVersionedIndexWithDataInBothIndexes() {
var version1Index = new VersionedEmployeeIndex(_configuration, 1);
await version1Index.DeleteAsync();
var version2Index = new VersionedEmployeeIndex(_configuration, 2);
await version2Index.DeleteAsync();
using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
await version1Index.ConfigureAsync();
Assert.True(_client.IndexExists(version1Index.VersionedName).Exists);
var version1Repository = new EmployeeRepository(version1Index.Employee);
var employee = await version1Repository.AddAsync(EmployeeGenerator.Default);
Assert.NotNull(employee?.Id);
await _client.RefreshAsync();
using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
await version2Index.ConfigureAsync();
Assert.True(_client.IndexExists(version2Index.VersionedName).Exists);
// swap the alias so we write to v1 and v2 and try to reindex.
await _client.AliasAsync(x => x
.Remove(a => a.Alias(version1Index.Name).Index(version1Index.VersionedName))
.Add(a => a.Alias(version2Index.Name).Index(version2Index.VersionedName)));
var version2Repository = new EmployeeRepository(version2Index.Employee);
await version2Repository.AddAsync(EmployeeGenerator.Generate());
await _client.RefreshAsync();
var countResponse = await _client.CountAsync(d => d.Index(version1Index.VersionedName));
_logger.Trace(() => countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(1, countResponse.Count);
countResponse = await _client.CountAsync(d => d.Index(version2Index.VersionedName));
_logger.Trace(() => countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(1, countResponse.Count);
// swap back the alias
await _client.AliasAsync(x => x
.Remove(a => a.Alias(version2Index.Name).Index(version2Index.VersionedName))
.Add(a => a.Alias(version1Index.Name).Index(version1Index.VersionedName)));
Assert.Equal(1, await version2Index.GetCurrentVersionAsync());
// alias should still point to the old version until reindex
var aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Alias(version2Index.Name));
Assert.True(aliasResponse.IsValid);
Assert.Equal(1, aliasResponse.Indices.Count);
Assert.Equal(version1Index.VersionedName, aliasResponse.Indices.First().Key);
await version2Index.ReindexAsync();
aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Alias(version2Index.Name));
Assert.True(aliasResponse.IsValid);
Assert.Equal(1, aliasResponse.Indices.Count);
Assert.Equal(version2Index.VersionedName, aliasResponse.Indices.First().Key);
Assert.Equal(2, await version1Index.GetCurrentVersionAsync());
Assert.Equal(2, await version2Index.GetCurrentVersionAsync());
countResponse = await _client.CountAsync(d => d.Index(version2Index.VersionedName));
_logger.Trace(() => countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(2, countResponse.Count);
Assert.False(_client.IndexExists(d => d.Index(version1Index.VersionedName)).Exists);
}
}
}
public async Task CanReindexVersionedIndexWithCorrectMappings() {
var version1Index = new VersionedEmployeeIndex(_configuration, 1);
await version1Index.DeleteAsync();
var version2Index = new VersionedEmployeeIndex(_configuration, 2);
version2Index.DiscardIndexesOnReindex = false;
await version2Index.DeleteAsync();
using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
await version1Index.ConfigureAsync();
var version1Repository = new EmployeeRepository(version1Index.Employee);
var utcNow = SystemClock.UtcNow;
var employee = await version1Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow));
Assert.NotNull(employee?.Id);
await _client.RefreshAsync();
using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
await version2Index.ConfigureAsync();
await version2Index.ReindexAsync();
var existsResponse = await _client.IndexExistsAsync(d => d.Index(version1Index.VersionedName));
_logger.Trace(() => existsResponse.GetRequest());
Assert.True(existsResponse.IsValid);
Assert.True(existsResponse.Exists);
var mappingResponse = await _client.GetMappingAsync<Employee>(m => m.Index(version1Index.VersionedName));
_logger.Trace(() => mappingResponse.GetRequest());
Assert.True(mappingResponse.IsValid);
Assert.NotNull(mappingResponse.Mappings);
existsResponse = await _client.IndexExistsAsync(d => d.Index(version2Index.VersionedName));
_logger.Trace(() => existsResponse.GetRequest());
Assert.True(existsResponse.IsValid);
Assert.True(existsResponse.Exists);
string version1Mappings = ToJson(mappingResponse.Mappings);
mappingResponse = await _client.GetMappingAsync<Employee>(m => m.Index(version1Index.VersionedName));
_logger.Trace(() => mappingResponse.GetRequest());
Assert.True(mappingResponse.IsValid);
Assert.NotNull(mappingResponse.Mappings);
Assert.Equal(version1Mappings, ToJson(mappingResponse.Mappings).Replace("-v2", "-v1"));
}
}
}
public async Task CanReindexVersionedIndex() {
var version1Index = new VersionedEmployeeIndex(_configuration, 1);
await version1Index.DeleteAsync();
var version2Index = new VersionedEmployeeIndex(_configuration, 2);
await version2Index.DeleteAsync();
using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
await version1Index.ConfigureAsync();
Assert.True(_client.IndexExists(version1Index.VersionedName).Exists);
var indexes = _client.GetIndicesPointingToAlias(version1Index.Name);
Assert.Equal(1, indexes.Count);
var aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Alias(version1Index.Name));
_logger.Trace(() => aliasResponse.GetRequest());
Assert.True(aliasResponse.IsValid);
Assert.Equal(1, aliasResponse.Indices.Count);
Assert.Equal(version1Index.VersionedName, aliasResponse.Indices.First().Key);
var version1Repository = new EmployeeRepository(version1Index.Employee);
var employee = await version1Repository.AddAsync(EmployeeGenerator.Default);
await _client.RefreshAsync();
Assert.NotNull(employee?.Id);
var countResponse = await _client.CountAsync(d => d.Index(version1Index.Name));
_logger.Trace(() => countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(1, countResponse.Count);
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
await version2Index.ConfigureAsync();
Assert.True(_client.IndexExists(version2Index.VersionedName).Exists);
// Make sure we can write to the index still. Should go to the old index until after the reindex is complete.
var version2Repository = new EmployeeRepository(version2Index.Employee);
await version2Repository.AddAsync(EmployeeGenerator.Generate());
await _client.RefreshAsync();
countResponse = await _client.CountAsync(d => d.Index(version1Index.VersionedName));
_logger.Trace(() => countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(2, countResponse.Count);
countResponse = await _client.CountAsync(d => d.Index(version2Index.VersionedName));
_logger.Trace(() => countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(0, countResponse.Count);
Assert.Equal(1, await version2Index.GetCurrentVersionAsync());
// alias should still point to the old version until reindex
aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Alias(version2Index.Name));
Assert.True(aliasResponse.IsValid);
Assert.Equal(1, aliasResponse.Indices.Count);
Assert.Equal(version1Index.VersionedName, aliasResponse.Indices.First().Key);
await version2Index.ReindexAsync();
aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Alias(version2Index.Name));
Assert.True(aliasResponse.IsValid);
Assert.Equal(1, aliasResponse.Indices.Count);
Assert.Equal(version2Index.VersionedName, aliasResponse.Indices.First().Key);
Assert.Equal(2, await version1Index.GetCurrentVersionAsync());
Assert.Equal(2, await version2Index.GetCurrentVersionAsync());
countResponse = await _client.CountAsync(d => d.Index(version2Index.VersionedName));
_logger.Trace(() => countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(2, countResponse.Count);
Assert.False(_client.IndexExists(d => d.Index(version1Index.VersionedName)).Exists);
employee = await version2Repository.AddAsync(EmployeeGenerator.Default);
await _client.RefreshAsync();
Assert.NotNull(employee?.Id);
countResponse = await _client.CountAsync(d => d.Index(version2Index.Name));
_logger.Trace(() => countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(3, countResponse.Count);
}
}
}
public async Task CanResumeReindex() {
const int numberOfEmployeesToCreate = 2000;
var version1Index = new VersionedEmployeeIndex(_configuration, 1);
await version1Index.DeleteAsync();
var version2Index = new VersionedEmployeeIndex(_configuration, 2);
await version2Index.DeleteAsync();
using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
await version1Index.ConfigureAsync();
Assert.True(_client.IndexExists(version1Index.VersionedName).Exists);
var version1Repository = new EmployeeRepository(version1Index.Employee);
await version1Repository.AddAsync(EmployeeGenerator.GenerateEmployees(numberOfEmployeesToCreate));
await _client.RefreshAsync();
var countResponse = await _client.CountAsync(d => d.Index(version1Index.Name));
_logger.Trace(() => countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(numberOfEmployeesToCreate, countResponse.Count);
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
await version2Index.ConfigureAsync();
Assert.True(_client.IndexExists(version2Index.VersionedName).Exists);
await Assert.ThrowsAsync<ApplicationException>(async () => await version2Index.ReindexAsync((progress, message) => {
_logger.Info("Reindex Progress {0}%: {1}", progress, message);
// TODO: Need to make this so it happens randomly in the middle of a batch
if (progress >= 45)
throw new ApplicationException("Random Error");
return Task.CompletedTask;
}));
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
await version2Index.ReindexAsync();
var aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Alias(version2Index.Name));
Assert.True(aliasResponse.IsValid);
Assert.Equal(1, aliasResponse.Indices.Count);
Assert.Equal(version2Index.VersionedName, aliasResponse.Indices.First().Key);
Assert.Equal(2, await version1Index.GetCurrentVersionAsync());
Assert.Equal(2, await version2Index.GetCurrentVersionAsync());
countResponse = await _client.CountAsync(d => d.Index(version2Index.VersionedName));
_logger.Trace(() => countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(numberOfEmployeesToCreate, countResponse.Count);
Assert.False(_client.IndexExists(d => d.Index(version1Index.VersionedName)).Exists);
}
}
}