public async Task MonthlyAliasMaxAgeAsync(DateTime utcNow)
{
using (TestSystemClock.Install()) {
SystemClock.Test.SetFixedTime(utcNow);
var index = new MonthlyEmployeeIndex(_configuration, 1)
{
MaxIndexAge = TimeSpan.FromDays(90)
};
await index.DeleteAsync();
using (new DisposableAction(() => index.DeleteAsync().GetAwaiter().GetResult())) {
await index.ConfigureAsync();
var repository = new EmployeeRepository(index.Employee);
var employee = await repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow), o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
var existsResponse = await _client.IndexExistsAsync(index.GetIndex(employee.CreatedUtc));
_logger.Trace(() => existsResponse.GetRequest());
Assert.True(existsResponse.IsValid);
Assert.True(existsResponse.Exists);
var aliasesResponse = await _client.GetAliasAsync(a => a.Index(index.GetIndex(employee.CreatedUtc)));
_logger.Trace(() => aliasesResponse.GetRequest());
Assert.True(aliasesResponse.IsValid);
Assert.Equal(1, aliasesResponse.Indices.Count);
var aliases = aliasesResponse.Indices.Values.Single().Select(s => s.Name).ToList();
aliases.Sort();
Assert.Equal(GetExpectedEmployeeMonthlyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases));
employee = await repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow.SubtractDays(2)), o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
existsResponse = await _client.IndexExistsAsync(index.GetIndex(employee.CreatedUtc));
_logger.Trace(() => existsResponse.GetRequest());
Assert.True(existsResponse.IsValid);
Assert.True(existsResponse.Exists);
aliasesResponse = await _client.GetAliasAsync(a => a.Index(index.GetIndex(employee.CreatedUtc)));
_logger.Trace(() => aliasesResponse.GetRequest());
Assert.True(aliasesResponse.IsValid);
Assert.Equal(1, aliasesResponse.Indices.Count);
aliases = aliasesResponse.Indices.Values.Single().Select(s => s.Name).ToList();
aliases.Sort();
Assert.Equal(GetExpectedEmployeeMonthlyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases));
employee = await repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow.SubtractDays(35)), o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
existsResponse = await _client.IndexExistsAsync(index.GetIndex(employee.CreatedUtc));
_logger.Trace(() => existsResponse.GetRequest());
Assert.True(existsResponse.IsValid);
Assert.True(existsResponse.Exists);
aliasesResponse = await _client.GetAliasAsync(a => a.Index(index.GetIndex(employee.CreatedUtc)));
_logger.Trace(() => aliasesResponse.GetRequest());
Assert.True(aliasesResponse.IsValid);
Assert.Equal(1, aliasesResponse.Indices.Count);
aliases = aliasesResponse.Indices.Values.Single().Select(s => s.Name).ToList();
aliases.Sort();
Assert.Equal(GetExpectedEmployeeMonthlyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases));
}
}
}
public SearchableReadOnlyRepositoryTests(ITestOutputHelper output) : base(output)
{
_employeeRepository = new EmployeeRepository(_configuration);
RemoveDataAsync().GetAwaiter().GetResult();
}
public async Task MaintainMonthlyIndexesAsync()
{
using (TestSystemClock.Install()) {
SystemClock.Test.SetFixedTime(new DateTime(2016, 8, 31, 0, 0, 0, DateTimeKind.Utc));
var index = new MonthlyEmployeeIndex(_configuration, 1)
{
MaxIndexAge = SystemClock.UtcNow.EndOfMonth() - SystemClock.UtcNow.SubtractMonths(4).StartOfMonth()
};
await index.DeleteAsync();
var utcNow = SystemClock.UtcNow;
using (new DisposableAction(() => index.DeleteAsync().GetAwaiter().GetResult())) {
await index.ConfigureAsync();
var repository = new EmployeeRepository(index.Employee);
for (int i = 0; i < 4; i++)
{
var created = utcNow.SubtractMonths(i);
var employee = await repository.AddAsync(EmployeeGenerator.Generate(createdUtc: created));
Assert.NotNull(employee?.Id);
Assert.Equal(1, await index.GetCurrentVersionAsync());
var existsResponse = await _client.IndexExistsAsync(index.GetIndex(employee.CreatedUtc));
_logger.Trace(() => existsResponse.GetRequest());
Assert.True(existsResponse.IsValid);
Assert.True(existsResponse.Exists);
var aliasesResponse = await _client.GetAliasAsync(a => a.Index(index.GetIndex(employee.CreatedUtc)));
_logger.Trace(() => aliasesResponse.GetRequest());
Assert.True(aliasesResponse.IsValid);
Assert.Equal(1, aliasesResponse.Indices.Count);
var aliases = aliasesResponse.Indices.Values.Single().Select(s => s.Name).ToList();
aliases.Sort();
Assert.Equal(GetExpectedEmployeeMonthlyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases));
}
await index.MaintainAsync();
for (int i = 0; i < 4; i++)
{
var created = utcNow.SubtractMonths(i);
var employee = await repository.AddAsync(EmployeeGenerator.Generate(createdUtc: created));
Assert.NotNull(employee?.Id);
Assert.Equal(1, await index.GetCurrentVersionAsync());
var existsResponse = await _client.IndexExistsAsync(index.GetIndex(employee.CreatedUtc));
_logger.Trace(() => existsResponse.GetRequest());
Assert.True(existsResponse.IsValid);
Assert.True(existsResponse.Exists);
var aliasesResponse = await _client.GetAliasAsync(a => a.Index(index.GetIndex(employee.CreatedUtc)));
_logger.Trace(() => aliasesResponse.GetRequest());
Assert.True(aliasesResponse.IsValid);
Assert.Equal(1, aliasesResponse.Indices.Count);
var aliases = aliasesResponse.Indices.Values.Single().Select(s => s.Name).ToList();
aliases.Sort();
Assert.Equal(GetExpectedEmployeeMonthlyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases));
}
}
}
}
public async Task CanReindexTimeSeriesIndexAsync()
{
var version1Index = new DailyEmployeeIndex(_configuration, 1);
await version1Index.DeleteAsync();
var version2Index = new DailyEmployeeIndex(_configuration, 2);
await version2Index.DeleteAsync();
using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
await version1Index.ConfigureAsync();
var version1Repository = new EmployeeRepository(version1Index.Employee);
var utcNow = SystemClock.UtcNow;
var employee = await version1Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow), o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
var aliasCountResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.Name));
_logger.LogTrace(aliasCountResponse.GetRequest());
Assert.True(aliasCountResponse.IsValid);
Assert.Equal(1, aliasCountResponse.Count);
var indexCountResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.GetIndex(utcNow)));
_logger.LogTrace(indexCountResponse.GetRequest());
Assert.True(indexCountResponse.IsValid);
Assert.Equal(1, indexCountResponse.Count);
indexCountResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.GetVersionedIndex(utcNow, 1)));
_logger.LogTrace(indexCountResponse.GetRequest());
Assert.True(indexCountResponse.IsValid);
Assert.Equal(1, indexCountResponse.Count);
using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
await version2Index.ConfigureAsync();
Assert.Equal(1, await version2Index.GetCurrentVersionAsync());
var version2Repository = new EmployeeRepository(version2Index.Employee);
// Make sure we write to the old index.
await version2Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow), o => o.ImmediateConsistency());
aliasCountResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.Name));
_logger.LogTrace(aliasCountResponse.GetRequest());
Assert.True(aliasCountResponse.IsValid);
Assert.Equal(2, aliasCountResponse.Count);
indexCountResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.GetVersionedIndex(utcNow, 1)));
_logger.LogTrace(indexCountResponse.GetRequest());
Assert.True(indexCountResponse.IsValid);
Assert.Equal(2, indexCountResponse.Count);
var existsResponse = await _client.IndexExistsAsync(version2Index.GetVersionedIndex(utcNow, 2));
_logger.LogTrace(existsResponse.GetRequest());
Assert.True(existsResponse.IsValid);
Assert.False(existsResponse.Exists);
// alias should still point to the old version until reindex
var aliasesResponse = await _client.GetAliasAsync(a => a.Index(version1Index.GetIndex(employee.CreatedUtc)));
_logger.LogTrace(aliasesResponse.GetRequest());
Assert.True(aliasesResponse.IsValid);
Assert.Equal(version1Index.GetVersionedIndex(employee.CreatedUtc, 1), aliasesResponse.Indices.Single().Key);
var aliases = aliasesResponse.Indices.Values.Single().Select(s => s.Name).ToList();
aliases.Sort();
Assert.Equal(GetExpectedEmployeeDailyAliases(version1Index, utcNow, employee.CreatedUtc), String.Join(", ", aliases));
await version2Index.ReindexAsync();
Assert.Equal(2, await version1Index.GetCurrentVersionAsync());
Assert.Equal(2, await version2Index.GetCurrentVersionAsync());
aliasesResponse = await _client.GetAliasAsync(a => a.Index(version1Index.GetIndex(employee.CreatedUtc)));
_logger.LogTrace(aliasesResponse.GetRequest());
Assert.True(aliasesResponse.IsValid);
Assert.Equal(version1Index.GetVersionedIndex(employee.CreatedUtc, 2), aliasesResponse.Indices.Single().Key);
aliases = aliasesResponse.Indices.Values.Single().Select(s => s.Name).ToList();
aliases.Sort();
Assert.Equal(GetExpectedEmployeeDailyAliases(version1Index, utcNow, employee.CreatedUtc), String.Join(", ", aliases));
existsResponse = await _client.IndexExistsAsync(version1Index.GetVersionedIndex(utcNow, 1));
_logger.LogTrace(existsResponse.GetRequest());
Assert.True(existsResponse.IsValid);
Assert.False(existsResponse.Exists);
existsResponse = await _client.IndexExistsAsync(version2Index.GetVersionedIndex(utcNow, 2));
_logger.LogTrace(existsResponse.GetRequest());
Assert.True(existsResponse.IsValid);
Assert.True(existsResponse.Exists);
}
}
}
public async Task CanResumeReindexAsync()
{
const int numberOfEmployeesToCreate = 2000;
var version1Index = new VersionedEmployeeIndex(_configuration, 1);
await version1Index.DeleteAsync();
var version2Index = new VersionedEmployeeIndex(_configuration, 2);
await version2Index.DeleteAsync();
using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
await version1Index.ConfigureAsync();
Assert.True(_client.IndexExists(version1Index.VersionedName).Exists);
var version1Repository = new EmployeeRepository(version1Index.Employee);
await version1Repository.AddAsync(EmployeeGenerator.GenerateEmployees(numberOfEmployeesToCreate), o => o.ImmediateConsistency());
var countResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.Name));
_logger.LogTrace(countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(numberOfEmployeesToCreate, countResponse.Count);
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
await version2Index.ConfigureAsync();
Assert.True(_client.IndexExists(version2Index.VersionedName).Exists);
// Throw error before second repass.
await Assert.ThrowsAsync <ApplicationException>(async() => await version2Index.ReindexAsync((progress, message) => {
_logger.LogInformation("Reindex Progress {0}%: {1}", progress, message);
if (progress == 91)
{
throw new ApplicationException("Random Error");
}
return(Task.CompletedTask);
}));
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
// Add a document and ensure it resumes from this document.
await version1Repository.AddAsync(EmployeeGenerator.Generate(ObjectId.GenerateNewId(SystemClock.UtcNow.AddMinutes(1)).ToString()), o => o.ImmediateConsistency());
await version2Index.ReindexAsync();
var aliasResponse = await _client.GetAliasAsync(d => d.Name(version2Index.Name));
Assert.True(aliasResponse.IsValid);
Assert.Equal(1, aliasResponse.Indices.Count);
Assert.Equal(version2Index.VersionedName, aliasResponse.Indices.First().Key);
Assert.Equal(2, await version1Index.GetCurrentVersionAsync());
Assert.Equal(2, await version2Index.GetCurrentVersionAsync());
countResponse = await _client.CountAsync <Employee>(d => d.Index(version2Index.VersionedName));
_logger.LogTrace(countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(numberOfEmployeesToCreate + 1, countResponse.Count);
Assert.False((await _client.IndexExistsAsync(version1Index.VersionedName)).Exists);
}
}
}
public async Task CanReindexVersionedIndexWithDataInBothIndexesAsync()
{
var version1Index = new VersionedEmployeeIndex(_configuration, 1);
await version1Index.DeleteAsync();
var version2Index = new VersionedEmployeeIndex(_configuration, 2);
await version2Index.DeleteAsync();
using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
await version1Index.ConfigureAsync();
Assert.True(_client.IndexExists(version1Index.VersionedName).Exists);
var version1Repository = new EmployeeRepository(version1Index.Employee);
var employee = await version1Repository.AddAsync(EmployeeGenerator.Default, o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
await version2Index.ConfigureAsync();
Assert.True(_client.IndexExists(version2Index.VersionedName).Exists);
// swap the alias so we write to v1 and v2 and try to reindex.
await _client.AliasAsync(x => x
.Remove(a => a.Alias(version1Index.Name).Index(version1Index.VersionedName))
.Add(a => a.Alias(version2Index.Name).Index(version2Index.VersionedName)));
var version2Repository = new EmployeeRepository(version2Index.Employee);
await version2Repository.AddAsync(EmployeeGenerator.Generate(), o => o.ImmediateConsistency());
var countResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.VersionedName));
_logger.LogTrace(countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(1, countResponse.Count);
countResponse = await _client.CountAsync <Employee>(d => d.Index(version2Index.VersionedName));
_logger.LogTrace(countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(1, countResponse.Count);
// swap back the alias
await _client.AliasAsync(x => x
.Remove(a => a.Alias(version2Index.Name).Index(version2Index.VersionedName))
.Add(a => a.Alias(version1Index.Name).Index(version1Index.VersionedName)));
Assert.Equal(1, await version2Index.GetCurrentVersionAsync());
// alias should still point to the old version until reindex
var aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Name(version2Index.Name));
Assert.True(aliasResponse.IsValid);
Assert.Equal(1, aliasResponse.Indices.Count);
Assert.Equal(version1Index.VersionedName, aliasResponse.Indices.First().Key);
await version2Index.ReindexAsync();
aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Name(version2Index.Name));
Assert.True(aliasResponse.IsValid);
Assert.Equal(1, aliasResponse.Indices.Count);
Assert.Equal(version2Index.VersionedName, aliasResponse.Indices.First().Key);
Assert.Equal(2, await version1Index.GetCurrentVersionAsync());
Assert.Equal(2, await version2Index.GetCurrentVersionAsync());
await _client.RefreshAsync(Indices.All);
countResponse = await _client.CountAsync <Employee>(d => d.Index(version2Index.VersionedName));
_logger.LogTrace(countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(2, countResponse.Count);
Assert.False((await _client.IndexExistsAsync(version1Index.VersionedName)).Exists);
}
}
}
public async Task CanReindexVersionedIndexWithDeletedDocsAsync()
{
var version1Index = new VersionedEmployeeIndex(_configuration, 1);
await version1Index.DeleteAsync();
var version2Index = new VersionedEmployeeIndex(_configuration, 2);
await version2Index.DeleteAsync();
using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
await version1Index.ConfigureAsync();
Assert.True(_client.IndexExists(version1Index.VersionedName).Exists);
var repository = new EmployeeRepository(version1Index.Employee);
var employee = await repository.AddAsync(EmployeeGenerator.Default, o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
await version2Index.ConfigureAsync();
Assert.True(_client.IndexExists(version2Index.VersionedName).Exists);
Assert.Equal(1, await version2Index.GetCurrentVersionAsync());
// alias should still point to the old version until reindex
var aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Name(version2Index.Name));
Assert.True(aliasResponse.IsValid);
Assert.Equal(1, aliasResponse.Indices.Count);
Assert.Equal(version1Index.VersionedName, aliasResponse.Indices.First().Key);
var countdown = new AsyncCountdownEvent(1);
var reindexTask = version2Index.ReindexAsync((progress, message) => {
_logger.LogInformation($"Reindex Progress {progress}%: {message}");
if (progress == 91)
{
countdown.Signal();
SystemClock.Sleep(1000);
}
return(Task.CompletedTask);
});
// Wait until the first reindex pass is done.
await countdown.WaitAsync();
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
await repository.RemoveAllAsync(o => o.ImmediateConsistency());
// Resume after everythings been indexed.
await reindexTask;
aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Name(version2Index.Name));
Assert.True(aliasResponse.IsValid, aliasResponse.GetErrorMessage());
Assert.Equal(1, aliasResponse.Indices.Count);
Assert.Equal(version2Index.VersionedName, aliasResponse.Indices.First().Key);
Assert.Equal(2, await version1Index.GetCurrentVersionAsync());
Assert.Equal(2, await version2Index.GetCurrentVersionAsync());
var countResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.VersionedName));
_logger.LogTrace(countResponse.GetRequest());
Assert.True(countResponse.ApiCall.HttpStatusCode == 404, countResponse.GetErrorMessage());
Assert.Equal(0, countResponse.Count);
countResponse = await _client.CountAsync <Employee>(d => d.Index(version2Index.VersionedName));
_logger.LogTrace(countResponse.GetRequest());
Assert.True(countResponse.IsValid, countResponse.GetErrorMessage());
Assert.Equal(1, countResponse.Count);
Assert.Equal(employee, await repository.GetByIdAsync(employee.Id));
Assert.False((await _client.IndexExistsAsync(version1Index.VersionedName)).Exists);
}
}
}
public async Task CanReindexVersionedIndexAsync()
{
var version1Index = new VersionedEmployeeIndex(_configuration, 1);
await version1Index.DeleteAsync();
var version2Index = new VersionedEmployeeIndex(_configuration, 2);
await version2Index.DeleteAsync();
using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
await version1Index.ConfigureAsync();
Assert.True(_client.IndexExists(version1Index.VersionedName).Exists);
var indexes = _client.GetIndicesPointingToAlias(version1Index.Name);
Assert.Single(indexes);
var aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Name(version1Index.Name));
_logger.LogTrace(aliasResponse.GetRequest());
Assert.True(aliasResponse.IsValid);
Assert.Equal(1, aliasResponse.Indices.Count);
Assert.Equal(version1Index.VersionedName, aliasResponse.Indices.First().Key);
var version1Repository = new EmployeeRepository(version1Index.Employee);
var employee = await version1Repository.AddAsync(EmployeeGenerator.Default, o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
var countResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.Name));
_logger.LogTrace(countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(1, countResponse.Count);
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
await version2Index.ConfigureAsync();
Assert.True(_client.IndexExists(version2Index.VersionedName).Exists);
// Make sure we can write to the index still. Should go to the old index until after the reindex is complete.
var version2Repository = new EmployeeRepository(version2Index.Employee);
await version2Repository.AddAsync(EmployeeGenerator.Generate(), o => o.ImmediateConsistency());
countResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.VersionedName));
_logger.LogTrace(countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(2, countResponse.Count);
countResponse = await _client.CountAsync <Employee>(d => d.Index(version2Index.VersionedName));
_logger.LogTrace(countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(0, countResponse.Count);
Assert.Equal(1, await version2Index.GetCurrentVersionAsync());
// alias should still point to the old version until reindex
aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Name(version2Index.Name));
Assert.True(aliasResponse.IsValid);
Assert.Equal(1, aliasResponse.Indices.Count);
Assert.Equal(version1Index.VersionedName, aliasResponse.Indices.First().Key);
await version2Index.ReindexAsync();
aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Name(version2Index.Name));
Assert.True(aliasResponse.IsValid);
Assert.Equal(1, aliasResponse.Indices.Count);
Assert.Equal(version2Index.VersionedName, aliasResponse.Indices.First().Key);
Assert.Equal(2, await version1Index.GetCurrentVersionAsync());
Assert.Equal(2, await version2Index.GetCurrentVersionAsync());
countResponse = await _client.CountAsync <Employee>(d => d.Index(version2Index.VersionedName));
_logger.LogTrace(countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(2, countResponse.Count);
Assert.False((await _client.IndexExistsAsync(version1Index.VersionedName)).Exists);
employee = await version2Repository.AddAsync(EmployeeGenerator.Default, o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
countResponse = await _client.CountAsync <Employee>(d => d.Index(version2Index.Name));
_logger.LogTrace(countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(3, countResponse.Count);
}
}
}
public async Task CanReindexVersionedIndexWithReindexScriptAsync()
{
var version1Index = new VersionedEmployeeIndex(_configuration, 1);
await version1Index.DeleteAsync();
var version20Index = new VersionedEmployeeIndex(_configuration, 20)
{
DiscardIndexesOnReindex = false
};
await version20Index.DeleteAsync();
var version21Index = new VersionedEmployeeIndex(_configuration, 21)
{
DiscardIndexesOnReindex = false
};
await version21Index.DeleteAsync();
using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
await version1Index.ConfigureAsync();
var version1Repository = new EmployeeRepository(version1Index.Employee);
var utcNow = SystemClock.UtcNow;
var employee = await version1Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow), o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
using (new DisposableAction(() => version20Index.DeleteAsync().GetAwaiter().GetResult())) {
await version20Index.ConfigureAsync();
await version20Index.ReindexAsync();
var version20Repository = new EmployeeRepository(version20Index.Employee);
var result = await version20Repository.GetByIdAsync(employee.Id);
Assert.Equal("scripted", result.CompanyName);
using (new DisposableAction(() => version21Index.DeleteAsync().GetAwaiter().GetResult())) {
await version21Index.ConfigureAsync();
await version21Index.ReindexAsync();
var version21Repository = new EmployeeRepository(version21Index.Employee);
result = await version21Repository.GetByIdAsync(employee.Id);
Assert.Equal("typed script", result.CompanyName);
}
}
}
using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
await version1Index.ConfigureAsync();
var version1Repository = new EmployeeRepository(version1Index.Employee);
var utcNow = SystemClock.UtcNow;
var employee = await version1Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow), o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
using (new DisposableAction(() => version21Index.DeleteAsync().GetAwaiter().GetResult())) {
await version21Index.ConfigureAsync();
await version21Index.ReindexAsync();
var version21Repository = new EmployeeRepository(version21Index.Employee);
var result = await version21Repository.GetByIdAsync(employee.Id);
Assert.Equal("typed script", result.CompanyName);
}
}
}
public async Task CanHandleReindexFailureAsync()
{
var version1Index = new VersionedEmployeeIndex(_configuration, 1);
await version1Index.DeleteAsync();
var version2Index = new VersionedEmployeeIndex(_configuration, 2);
await version2Index.DeleteAsync();
using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
await version1Index.ConfigureAsync();
Assert.True(_client.IndexExists(version1Index.VersionedName).Exists);
var version1Repository = new EmployeeRepository(version1Index.Employee);
await version1Repository.AddAsync(EmployeeGenerator.Generate(), o => o.ImmediateConsistency());
var countResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.Name));
_logger.LogTrace(countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(1, countResponse.Count);
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
//Create invalid mappings
var response = await _client.CreateIndexAsync(version2Index.VersionedName, d => d.Mappings(m => m.Map <Employee>(map => map
.Dynamic(false)
.Properties(p => p
.Number(f => f.Name(e => e.Id))
))));
_logger.LogTrace(response.GetRequest());
Assert.True(_client.IndexExists(version2Index.VersionedName).Exists);
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
await version2Index.ReindexAsync();
var aliasResponse = await _client.GetAliasAsync(d => d.Name(version2Index.Name));
Assert.True(aliasResponse.IsValid);
Assert.Equal(1, aliasResponse.Indices.Count);
Assert.True(aliasResponse.Indices.ContainsKey(version1Index.VersionedName));
var indexResponse = await _client.CatIndicesAsync(d => d.Index(Indices.Index("employees-*")));
Assert.NotNull(indexResponse.Records.FirstOrDefault(r => r.Index == version1Index.VersionedName));
Assert.NotNull(indexResponse.Records.FirstOrDefault(r => r.Index == version2Index.VersionedName));
Assert.NotNull(indexResponse.Records.FirstOrDefault(r => r.Index == $"{version2Index.VersionedName}-error"));
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
Assert.Equal(1, await version2Index.GetCurrentVersionAsync());
countResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.VersionedName));
_logger.LogTrace(countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(1, countResponse.Count);
countResponse = await _client.CountAsync <Employee>(d => d.Index(version2Index.VersionedName));
_logger.LogTrace(countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(0, countResponse.Count);
countResponse = await _client.CountAsync <object>(d => d.Index($"{version2Index.VersionedName}-error"));
_logger.LogTrace(countResponse.GetRequest());
Assert.True(countResponse.IsValid);
Assert.Equal(1, countResponse.Count);
}
}
}
public VersionedTests(ITestOutputHelper output) : base(output)
{
_employeeRepository = new EmployeeRepository(_configuration);
RemoveDataAsync().GetAwaiter().GetResult();
}
public async Task MaintainDailyIndexesAsync()
{
using (TestSystemClock.Install()) {
var index = new DailyEmployeeIndex(_configuration, 1);
await index.DeleteAsync();
using (new DisposableAction(() => index.DeleteAsync().GetAwaiter().GetResult())) {
await index.ConfigureAsync();
var repository = new EmployeeRepository(index.Employee);
SystemClock.Test.SetFixedTime(DateTime.UtcNow.Subtract(TimeSpan.FromDays(15)));
var employee = await repository.AddAsync(EmployeeGenerator.Generate(createdUtc: SystemClock.UtcNow), o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
await index.MaintainAsync();
Assert.Equal(1, await index.GetCurrentVersionAsync());
var existsResponse = await _client.IndexExistsAsync(index.GetIndex(employee.CreatedUtc));
_logger.LogTraceRequest(existsResponse);
Assert.True(existsResponse.IsValid);
Assert.True(existsResponse.Exists);
var aliasesResponse = await _client.GetAliasAsync(a => a.Index(index.GetIndex(employee.CreatedUtc)));
_logger.LogTraceRequest(aliasesResponse);
Assert.True(aliasesResponse.IsValid);
Assert.Equal(1, aliasesResponse.Indices.Count);
var aliases = aliasesResponse.Indices.Values.Single().Select(s => s.Name).ToList();
aliases.Sort();
Assert.Equal(GetExpectedEmployeeDailyAliases(index, SystemClock.UtcNow, employee.CreatedUtc), String.Join(", ", aliases));
SystemClock.Test.SetFixedTime(DateTime.UtcNow.Subtract(TimeSpan.FromDays(9)));
index.MaxIndexAge = TimeSpan.FromDays(10);
await index.MaintainAsync();
existsResponse = await _client.IndexExistsAsync(index.GetIndex(employee.CreatedUtc));
_logger.LogTraceRequest(existsResponse);
Assert.True(existsResponse.IsValid);
Assert.True(existsResponse.Exists);
aliasesResponse = await _client.GetAliasAsync(a => a.Index(index.GetIndex(employee.CreatedUtc)));
_logger.LogTraceRequest(aliasesResponse);
Assert.True(aliasesResponse.IsValid);
Assert.Equal(1, aliasesResponse.Indices.Count);
aliases = aliasesResponse.Indices.Values.Single().Select(s => s.Name).ToList();
aliases.Sort();
Assert.Equal(GetExpectedEmployeeDailyAliases(index, SystemClock.UtcNow, employee.CreatedUtc), String.Join(", ", aliases));
SystemClock.Test.SetFixedTime(DateTime.UtcNow);
await index.MaintainAsync();
existsResponse = await _client.IndexExistsAsync(index.GetIndex(employee.CreatedUtc));
_logger.LogTraceRequest(existsResponse);
Assert.True(existsResponse.IsValid);
Assert.False(existsResponse.Exists);
aliasesResponse = await _client.GetAliasAsync(a => a.Index(index.GetIndex(employee.CreatedUtc)));
_logger.LogTraceRequest(aliasesResponse);
Assert.False(aliasesResponse.IsValid);
}
}
}
public async Task DailyAliasMaxAgeAsync(DateTime utcNow)
{
using (TestSystemClock.Install()) {
TestSystemClock.SetFrozenTime(utcNow);
var index = new DailyEmployeeIndex(_configuration, 1)
{
MaxIndexAge = TimeSpan.FromDays(45)
};
await index.DeleteAsync();
using (new DisposableAction(() => index.DeleteAsync().GetAwaiter().GetResult())) {
await index.ConfigureAsync();
IEmployeeRepository version1Repository = new EmployeeRepository(index);
var employee = await version1Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow), o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc));
_logger.LogRequest(existsResponse);
Assert.True(existsResponse.ApiCall.Success);
Assert.True(existsResponse.Exists);
var aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc));
_logger.LogRequest(aliasesResponse);
Assert.True(aliasesResponse.IsValid);
Assert.Equal(1, aliasesResponse.Indices.Count);
var aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList();
aliases.Sort();
Assert.Equal(GetExpectedEmployeeDailyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases));
employee = await version1Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow.SubtractDays(2)), o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc));
_logger.LogRequest(existsResponse);
Assert.True(existsResponse.ApiCall.Success);
Assert.True(existsResponse.Exists);
aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc));
_logger.LogRequest(aliasesResponse);
Assert.True(aliasesResponse.IsValid);
Assert.Equal(1, aliasesResponse.Indices.Count);
aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList();
aliases.Sort();
Assert.Equal(GetExpectedEmployeeDailyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases));
employee = await version1Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow.SubtractDays(35)), o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc));
_logger.LogRequest(existsResponse);
Assert.True(existsResponse.ApiCall.Success);
Assert.True(existsResponse.Exists);
aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc));
_logger.LogRequest(aliasesResponse);
Assert.True(aliasesResponse.IsValid);
Assert.Equal(1, aliasesResponse.Indices.Count);
aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList();
aliases.Sort();
Assert.Equal(GetExpectedEmployeeDailyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases));
}
}
}
public async Task CanReindexVersionedIndexWithUpdatedDocsAsync()
{
var version1Index = new VersionedEmployeeIndex(_configuration, 1);
await version1Index.DeleteAsync();
var version2Index = new VersionedEmployeeIndex(_configuration, 2);
await version2Index.DeleteAsync();
using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
await version1Index.ConfigureAsync();
Assert.True((await _client.IndexExistsAsync(version1Index.VersionedName)).Exists);
var repository = new EmployeeRepository(version1Index.Employee);
var employee = await repository.AddAsync(EmployeeGenerator.Default, o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
await version2Index.ConfigureAsync();
Assert.True((await _client.IndexExistsAsync(version2Index.VersionedName)).Exists);
Assert.Equal(1, await version2Index.GetCurrentVersionAsync());
// alias should still point to the old version until reindex
var aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Name(version2Index.Name));
_logger.LogTraceRequest(aliasResponse);
Assert.True(aliasResponse.IsValid);
Assert.Equal(1, aliasResponse.Indices.Count);
Assert.Equal(version1Index.VersionedName, aliasResponse.Indices.First().Key);
var countdown = new AsyncCountdownEvent(1);
var reindexTask = version2Index.ReindexAsync(async(progress, message) => {
_logger.LogInformation($"Reindex Progress {progress}%: {message}");
if (progress == 91)
{
countdown.Signal();
await SystemClock.SleepAsync(1000);
}
});
// Wait until the first reindex pass is done.
await countdown.WaitAsync();
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
await repository.AddAsync(EmployeeGenerator.Generate(createdUtc: SystemClock.UtcNow));
employee.Name = "Updated";
await repository.SaveAsync(employee);
// Resume after everythings been indexed.
await reindexTask;
aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Name(version2Index.Name));
Assert.True(aliasResponse.IsValid);
Assert.Equal(1, aliasResponse.Indices.Count);
Assert.Equal(version2Index.VersionedName, aliasResponse.Indices.First().Key);
Assert.Equal(2, await version1Index.GetCurrentVersionAsync());
Assert.Equal(2, await version2Index.GetCurrentVersionAsync());
await _client.RefreshAsync(Indices.All);
var countResponse = await _client.CountAsync <Employee>(d => d.Index(version2Index.VersionedName));
_logger.LogTraceRequest(countResponse);
Assert.True(countResponse.IsValid);
Assert.Equal(2, countResponse.Count);
var result = await repository.GetByIdAsync(employee.Id);
Assert.Equal(ToJson(employee), ToJson(result));
Assert.False((await _client.IndexExistsAsync(version1Index.VersionedName)).Exists);
}
}
}