public async Task CanReindexTimeSeriesIndexWithCorrectMappingsAsync()
{
var version1Index = new DailyEmployeeIndex(_configuration, 1);
await version1Index.DeleteAsync();
var version2Index = new DailyEmployeeIndex(_configuration, 2)
{
DiscardIndexesOnReindex = false
};
await version2Index.DeleteAsync();
using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
await version1Index.ConfigureAsync();
IEmployeeRepository version1Repository = new EmployeeRepository(version1Index);
var utcNow = SystemClock.UtcNow;
var employee = await version1Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow), o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
await version2Index.ConfigureAsync();
await version2Index.ReindexAsync();
var existsResponse = await _client.Indices.ExistsAsync(version1Index.GetVersionedIndex(utcNow, 1));
_logger.LogRequest(existsResponse);
Assert.True(existsResponse.ApiCall.Success);
Assert.True(existsResponse.Exists);
var indexV1 = version1Index.GetVersionedIndex(utcNow, 1);
var mappingResponse = await _client.Indices.GetMappingAsync <Employee>(m => m.Index(indexV1));
_logger.LogRequest(mappingResponse);
Assert.True(mappingResponse.IsValid);
var mappingsV1 = mappingResponse.Indices[indexV1];
Assert.NotNull(mappingsV1);
string version1Mappings = ToJson(mappingsV1);
var indexV2 = version2Index.GetVersionedIndex(utcNow, 2);
existsResponse = await _client.Indices.ExistsAsync(indexV2);
_logger.LogRequest(existsResponse);
Assert.True(existsResponse.ApiCall.Success);
Assert.True(existsResponse.Exists);
mappingResponse = await _client.Indices.GetMappingAsync <Employee>(m => m.Index(indexV2));
_logger.LogRequest(mappingResponse);
Assert.True(mappingResponse.IsValid);
var mappingsV2 = mappingResponse.Indices[indexV2];
Assert.NotNull(mappingsV2);
string version2Mappings = ToJson(mappingsV2);
Assert.Equal(version1Mappings, version2Mappings);
}
}
}
public async Task CanCreateAndDeleteDailyIndex()
{
var index = new DailyEmployeeIndex(_configuration, 1);
await index.ConfigureAsync();
var todayDate = SystemClock.Now;
var yesterdayDate = SystemClock.Now.SubtractDays(1);
var todayIndex = index.GetIndex(todayDate);
var yesterdayIndex = index.GetIndex(yesterdayDate);
await index.EnsureIndexAsync(todayDate);
await index.EnsureIndexAsync(yesterdayDate);
var existsResponse = await _client.Indices.ExistsAsync(todayIndex);
_logger.LogRequest(existsResponse);
Assert.True(existsResponse.ApiCall.Success);
Assert.True(existsResponse.Exists);
existsResponse = await _client.Indices.ExistsAsync(yesterdayIndex);
_logger.LogRequest(existsResponse);
Assert.True(existsResponse.ApiCall.Success);
Assert.True(existsResponse.Exists);
await index.DeleteAsync();
existsResponse = await _client.Indices.ExistsAsync(todayIndex);
_logger.LogRequest(existsResponse);
Assert.True(existsResponse.ApiCall.Success);
Assert.False(existsResponse.Exists);
existsResponse = await _client.Indices.ExistsAsync(yesterdayIndex);
_logger.LogRequest(existsResponse);
Assert.True(existsResponse.ApiCall.Success);
Assert.False(existsResponse.Exists);
}
public async Task DailyIndexMaxAgeAsync(DateTime utcNow)
{
using (TestSystemClock.Install()) {
SystemClock.Test.SetFixedTime(utcNow);
var index = new DailyEmployeeIndex(_configuration, 1)
{
MaxIndexAge = TimeSpan.FromDays(1)
};
await index.DeleteAsync();
using (new DisposableAction(() => index.DeleteAsync().GetAwaiter().GetResult())) {
await index.ConfigureAsync();
await index.EnsureIndexAsync(utcNow);
var existsResponse = await _client.IndexExistsAsync(index.GetIndex(utcNow));
_logger.Trace(() => existsResponse.GetRequest());
Assert.True(existsResponse.IsValid);
Assert.True(existsResponse.Exists);
await index.EnsureIndexAsync(utcNow.SubtractDays(1));
existsResponse = await _client.IndexExistsAsync(index.GetIndex(utcNow.SubtractDays(1)));
_logger.Trace(() => existsResponse.GetRequest());
Assert.True(existsResponse.IsValid);
Assert.True(existsResponse.Exists);
await Assert.ThrowsAsync <ArgumentException>(async() => await index.EnsureIndexAsync(utcNow.SubtractDays(2)));
existsResponse = await _client.IndexExistsAsync(index.GetIndex(utcNow.SubtractDays(2)));
_logger.Trace(() => existsResponse.GetRequest());
Assert.True(existsResponse.IsValid);
Assert.False(existsResponse.Exists);
}
}
}
public async Task CanCreateDailyAliasesAsync(DateTime utcNow)
{
using (TestSystemClock.Install()) {
SystemClock.Test.SetFixedTime(utcNow);
var index = new DailyEmployeeIndex(_configuration, 1);
await index.DeleteAsync();
using (new DisposableAction(() => index.DeleteAsync().GetAwaiter().GetResult())) {
await index.ConfigureAsync();
var repository = new EmployeeRepository(index.Employee);
for (int i = 0; i < 35; i += 5)
{
var employee = await repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow.SubtractDays(i)));
Assert.NotNull(employee?.Id);
Assert.Equal(1, await index.GetCurrentVersionAsync());
var existsResponse = await _client.IndexExistsAsync(index.GetIndex(employee.CreatedUtc));
_logger.Trace(() => existsResponse.GetRequest());
Assert.True(existsResponse.IsValid);
Assert.True(existsResponse.Exists);
var aliasesResponse = await _client.GetAliasAsync(a => a.Index(index.GetIndex(employee.CreatedUtc)));
_logger.Trace(() => aliasesResponse.GetRequest());
Assert.True(aliasesResponse.IsValid);
Assert.Equal(1, aliasesResponse.Indices.Count);
var aliases = aliasesResponse.Indices.Values.Single().Select(s => s.Name).ToList();
aliases.Sort();
Assert.Equal(GetExpectedEmployeeDailyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases));
}
}
}
}
public async Task CanReindexTimeSeriesIndexAsync()
{
var version1Index = new DailyEmployeeIndex(_configuration, 1);
await version1Index.DeleteAsync();
var version2Index = new DailyEmployeeIndex(_configuration, 2);
await version2Index.DeleteAsync();
using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
await version1Index.ConfigureAsync();
IEmployeeRepository version1Repository = new EmployeeRepository(version1Index);
var utcNow = SystemClock.UtcNow;
var employee = await version1Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow), o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
var aliasCountResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.Name));
_logger.LogRequest(aliasCountResponse);
Assert.True(aliasCountResponse.IsValid);
Assert.Equal(1, aliasCountResponse.Count);
var indexCountResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.GetIndex(utcNow)));
_logger.LogRequest(indexCountResponse);
Assert.True(indexCountResponse.IsValid);
Assert.Equal(1, indexCountResponse.Count);
indexCountResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.GetVersionedIndex(utcNow, 1)));
_logger.LogRequest(indexCountResponse);
Assert.True(indexCountResponse.IsValid);
Assert.Equal(1, indexCountResponse.Count);
using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
await version2Index.ConfigureAsync();
Assert.Equal(1, await version2Index.GetCurrentVersionAsync());
IEmployeeRepository version2Repository = new EmployeeRepository(version2Index);
// Make sure we write to the old index.
await version2Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow), o => o.ImmediateConsistency());
aliasCountResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.Name));
_logger.LogRequest(aliasCountResponse);
Assert.True(aliasCountResponse.IsValid);
Assert.Equal(2, aliasCountResponse.Count);
indexCountResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.GetVersionedIndex(utcNow, 1)));
_logger.LogRequest(indexCountResponse);
Assert.True(indexCountResponse.IsValid);
Assert.Equal(2, indexCountResponse.Count);
var existsResponse = await _client.Indices.ExistsAsync(version2Index.GetVersionedIndex(utcNow, 2));
_logger.LogRequest(existsResponse);
Assert.True(existsResponse.ApiCall.Success);
Assert.False(existsResponse.Exists);
// alias should still point to the old version until reindex
var aliasesResponse = await _client.Indices.GetAliasAsync(version1Index.GetIndex(employee.CreatedUtc));
_logger.LogRequest(aliasesResponse);
Assert.True(aliasesResponse.IsValid);
Assert.Equal(version1Index.GetVersionedIndex(employee.CreatedUtc, 1), aliasesResponse.Indices.Single().Key);
var aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList();
aliases.Sort();
Assert.Equal(GetExpectedEmployeeDailyAliases(version1Index, utcNow, employee.CreatedUtc), String.Join(", ", aliases));
await version2Index.ReindexAsync();
Assert.Equal(2, await version1Index.GetCurrentVersionAsync());
Assert.Equal(2, await version2Index.GetCurrentVersionAsync());
aliasesResponse = await _client.Indices.GetAliasAsync(version1Index.GetIndex(employee.CreatedUtc));
_logger.LogRequest(aliasesResponse);
Assert.True(aliasesResponse.IsValid);
Assert.Equal(version1Index.GetVersionedIndex(employee.CreatedUtc, 2), aliasesResponse.Indices.Single().Key);
aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList();
aliases.Sort();
Assert.Equal(GetExpectedEmployeeDailyAliases(version1Index, utcNow, employee.CreatedUtc), String.Join(", ", aliases));
existsResponse = await _client.Indices.ExistsAsync(version1Index.GetVersionedIndex(utcNow, 1));
_logger.LogRequest(existsResponse);
Assert.True(existsResponse.ApiCall.Success);
Assert.False(existsResponse.Exists);
existsResponse = await _client.Indices.ExistsAsync(version2Index.GetVersionedIndex(utcNow, 2));
_logger.LogRequest(existsResponse);
Assert.True(existsResponse.ApiCall.Success);
Assert.True(existsResponse.Exists);
}
}
}
public async Task DailyAliasMaxAgeAsync(DateTime utcNow)
{
using (TestSystemClock.Install()) {
SystemClock.Test.SetFixedTime(utcNow);
var index = new DailyEmployeeIndex(_configuration, 1)
{
MaxIndexAge = TimeSpan.FromDays(45)
};
await index.DeleteAsync();
using (new DisposableAction(() => index.DeleteAsync().GetAwaiter().GetResult())) {
await index.ConfigureAsync();
var version1Repository = new EmployeeRepository(index.Employee);
var employee = await version1Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow), o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
var existsResponse = await _client.IndexExistsAsync(index.GetIndex(employee.CreatedUtc));
_logger.Trace(() => existsResponse.GetRequest());
Assert.True(existsResponse.IsValid);
Assert.True(existsResponse.Exists);
var aliasesResponse = await _client.GetAliasAsync(a => a.Index(index.GetIndex(employee.CreatedUtc)));
_logger.Trace(() => aliasesResponse.GetRequest());
Assert.True(aliasesResponse.IsValid);
Assert.Equal(1, aliasesResponse.Indices.Count);
var aliases = aliasesResponse.Indices.Values.Single().Select(s => s.Name).ToList();
aliases.Sort();
Assert.Equal(GetExpectedEmployeeDailyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases));
employee = await version1Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow.SubtractDays(2)), o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
existsResponse = await _client.IndexExistsAsync(index.GetIndex(employee.CreatedUtc));
_logger.Trace(() => existsResponse.GetRequest());
Assert.True(existsResponse.IsValid);
Assert.True(existsResponse.Exists);
aliasesResponse = await _client.GetAliasAsync(a => a.Index(index.GetIndex(employee.CreatedUtc)));
_logger.Trace(() => aliasesResponse.GetRequest());
Assert.True(aliasesResponse.IsValid);
Assert.Equal(1, aliasesResponse.Indices.Count);
aliases = aliasesResponse.Indices.Values.Single().Select(s => s.Name).ToList();
aliases.Sort();
Assert.Equal(GetExpectedEmployeeDailyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases));
employee = await version1Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow.SubtractDays(35)), o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
existsResponse = await _client.IndexExistsAsync(index.GetIndex(employee.CreatedUtc));
_logger.Trace(() => existsResponse.GetRequest());
Assert.True(existsResponse.IsValid);
Assert.True(existsResponse.Exists);
aliasesResponse = await _client.GetAliasAsync(a => a.Index(index.GetIndex(employee.CreatedUtc)));
_logger.Trace(() => aliasesResponse.GetRequest());
Assert.True(aliasesResponse.IsValid);
Assert.Equal(1, aliasesResponse.Indices.Count);
aliases = aliasesResponse.Indices.Values.Single().Select(s => s.Name).ToList();
aliases.Sort();
Assert.Equal(GetExpectedEmployeeDailyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases));
}
}
}
public async Task MaintainWillCreateAliasesOnTimeSeriesIndexAsync()
{
using (TestSystemClock.Install()) {
SystemClock.Test.SetFixedTime(SystemClock.UtcNow);
var version1Index = new DailyEmployeeIndex(_configuration, 1);
await version1Index.DeleteAsync();
var version2Index = new DailyEmployeeIndex(_configuration, 2);
await version2Index.DeleteAsync();
// Indexes don't exist yet so the current version will be the index version.
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
Assert.Equal(2, await version2Index.GetCurrentVersionAsync());
using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
await version1Index.ConfigureAsync();
await version1Index.EnsureIndexAsync(SystemClock.UtcNow);
Assert.True(_client.IndexExists(version1Index.GetVersionedIndex(SystemClock.UtcNow)).Exists);
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
// delete all aliases
await _configuration.Cache.RemoveAllAsync();
await DeleteAliasesAsync(version1Index.GetVersionedIndex(SystemClock.UtcNow));
using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
await version2Index.ConfigureAsync();
await version2Index.EnsureIndexAsync(SystemClock.UtcNow);
Assert.True(_client.IndexExists(version2Index.GetVersionedIndex(SystemClock.UtcNow)).Exists);
Assert.Equal(2, await version2Index.GetCurrentVersionAsync());
// delete all aliases
await _configuration.Cache.RemoveAllAsync();
await DeleteAliasesAsync(version2Index.GetVersionedIndex(SystemClock.UtcNow));
await _client.RefreshAsync(Indices.All);
var aliasesResponse = await _client.GetAliasAsync(a => a.Index($"{version1Index.GetVersionedIndex(SystemClock.UtcNow)},{version2Index.GetVersionedIndex(SystemClock.UtcNow)}"));
Assert.Equal(0, aliasesResponse.Indices.SelectMany(i => i.Value).Count());
// Indexes exist but no alias so the oldest index version will be used.
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
Assert.Equal(1, await version2Index.GetCurrentVersionAsync());
await version1Index.MaintainAsync();
aliasesResponse = await _client.GetAliasAsync(a => a.Index(version1Index.GetVersionedIndex(SystemClock.UtcNow)));
Assert.Equal(version1Index.Aliases.Count + 1, aliasesResponse.Indices.Single().Value.Count);
aliasesResponse = await _client.GetAliasAsync(a => a.Index(version2Index.GetVersionedIndex(SystemClock.UtcNow)));
Assert.Equal(0, aliasesResponse.Indices.Single().Value.Count);
Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
Assert.Equal(1, await version2Index.GetCurrentVersionAsync());
}
}
}
}
public async Task MaintainDailyIndexesAsync()
{
using (TestSystemClock.Install()) {
var index = new DailyEmployeeIndex(_configuration, 1);
await index.DeleteAsync();
using (new DisposableAction(() => index.DeleteAsync().GetAwaiter().GetResult())) {
await index.ConfigureAsync();
IEmployeeRepository repository = new EmployeeRepository(index);
TestSystemClock.SetFrozenTime(DateTime.UtcNow.Subtract(TimeSpan.FromDays(15)));
var employee = await repository.AddAsync(EmployeeGenerator.Generate(createdUtc: SystemClock.UtcNow), o => o.ImmediateConsistency());
Assert.NotNull(employee?.Id);
await index.MaintainAsync();
Assert.Equal(1, await index.GetCurrentVersionAsync());
var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc));
_logger.LogRequest(existsResponse);
Assert.True(existsResponse.ApiCall.Success);
Assert.True(existsResponse.Exists);
var aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc));
_logger.LogRequest(aliasesResponse);
Assert.True(aliasesResponse.IsValid);
Assert.Equal(1, aliasesResponse.Indices.Count);
var aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList();
aliases.Sort();
Assert.Equal(GetExpectedEmployeeDailyAliases(index, SystemClock.UtcNow, employee.CreatedUtc), String.Join(", ", aliases));
TestSystemClock.SetFrozenTime(DateTime.UtcNow.Subtract(TimeSpan.FromDays(9)));
index.MaxIndexAge = TimeSpan.FromDays(10);
await index.MaintainAsync();
existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc));
_logger.LogRequest(existsResponse);
Assert.True(existsResponse.ApiCall.Success);
Assert.True(existsResponse.Exists);
aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc));
_logger.LogRequest(aliasesResponse);
Assert.True(aliasesResponse.IsValid);
Assert.Equal(1, aliasesResponse.Indices.Count);
aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList();
aliases.Sort();
Assert.Equal(GetExpectedEmployeeDailyAliases(index, SystemClock.UtcNow, employee.CreatedUtc), String.Join(", ", aliases));
TestSystemClock.SetFrozenTime(DateTime.UtcNow);
await index.MaintainAsync();
existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc));
_logger.LogRequest(existsResponse);
Assert.True(existsResponse.ApiCall.Success);
Assert.False(existsResponse.Exists);
aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc));
_logger.LogRequest(aliasesResponse);
Assert.False(aliasesResponse.IsValid);
}
}
}
