public async Task CanReindexTimeSeriesIndexWithCorrectMappingsAsync() { var version1Index = new DailyEmployeeIndex(_configuration, 1); await version1Index.DeleteAsync(); var version2Index = new DailyEmployeeIndex(_configuration, 2) { DiscardIndexesOnReindex = false }; await version2Index.DeleteAsync(); using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) { await version1Index.ConfigureAsync(); IEmployeeRepository version1Repository = new EmployeeRepository(version1Index); var utcNow = SystemClock.UtcNow; var employee = await version1Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow), o => o.ImmediateConsistency()); Assert.NotNull(employee?.Id); using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) { await version2Index.ConfigureAsync(); await version2Index.ReindexAsync(); var existsResponse = await _client.Indices.ExistsAsync(version1Index.GetVersionedIndex(utcNow, 1)); _logger.LogRequest(existsResponse); Assert.True(existsResponse.ApiCall.Success); Assert.True(existsResponse.Exists); var indexV1 = version1Index.GetVersionedIndex(utcNow, 1); var mappingResponse = await _client.Indices.GetMappingAsync <Employee>(m => m.Index(indexV1)); _logger.LogRequest(mappingResponse); Assert.True(mappingResponse.IsValid); var mappingsV1 = mappingResponse.Indices[indexV1]; Assert.NotNull(mappingsV1); string version1Mappings = ToJson(mappingsV1); var indexV2 = version2Index.GetVersionedIndex(utcNow, 2); existsResponse = await _client.Indices.ExistsAsync(indexV2); _logger.LogRequest(existsResponse); Assert.True(existsResponse.ApiCall.Success); Assert.True(existsResponse.Exists); mappingResponse = await _client.Indices.GetMappingAsync <Employee>(m => m.Index(indexV2)); _logger.LogRequest(mappingResponse); Assert.True(mappingResponse.IsValid); var mappingsV2 = mappingResponse.Indices[indexV2]; Assert.NotNull(mappingsV2); string version2Mappings = ToJson(mappingsV2); Assert.Equal(version1Mappings, version2Mappings); } } }
public async Task CanCreateAndDeleteDailyIndex() { var index = new DailyEmployeeIndex(_configuration, 1); await index.ConfigureAsync(); var todayDate = SystemClock.Now; var yesterdayDate = SystemClock.Now.SubtractDays(1); var todayIndex = index.GetIndex(todayDate); var yesterdayIndex = index.GetIndex(yesterdayDate); await index.EnsureIndexAsync(todayDate); await index.EnsureIndexAsync(yesterdayDate); var existsResponse = await _client.Indices.ExistsAsync(todayIndex); _logger.LogRequest(existsResponse); Assert.True(existsResponse.ApiCall.Success); Assert.True(existsResponse.Exists); existsResponse = await _client.Indices.ExistsAsync(yesterdayIndex); _logger.LogRequest(existsResponse); Assert.True(existsResponse.ApiCall.Success); Assert.True(existsResponse.Exists); await index.DeleteAsync(); existsResponse = await _client.Indices.ExistsAsync(todayIndex); _logger.LogRequest(existsResponse); Assert.True(existsResponse.ApiCall.Success); Assert.False(existsResponse.Exists); existsResponse = await _client.Indices.ExistsAsync(yesterdayIndex); _logger.LogRequest(existsResponse); Assert.True(existsResponse.ApiCall.Success); Assert.False(existsResponse.Exists); }
public async Task DailyIndexMaxAgeAsync(DateTime utcNow) { using (TestSystemClock.Install()) { SystemClock.Test.SetFixedTime(utcNow); var index = new DailyEmployeeIndex(_configuration, 1) { MaxIndexAge = TimeSpan.FromDays(1) }; await index.DeleteAsync(); using (new DisposableAction(() => index.DeleteAsync().GetAwaiter().GetResult())) { await index.ConfigureAsync(); await index.EnsureIndexAsync(utcNow); var existsResponse = await _client.IndexExistsAsync(index.GetIndex(utcNow)); _logger.Trace(() => existsResponse.GetRequest()); Assert.True(existsResponse.IsValid); Assert.True(existsResponse.Exists); await index.EnsureIndexAsync(utcNow.SubtractDays(1)); existsResponse = await _client.IndexExistsAsync(index.GetIndex(utcNow.SubtractDays(1))); _logger.Trace(() => existsResponse.GetRequest()); Assert.True(existsResponse.IsValid); Assert.True(existsResponse.Exists); await Assert.ThrowsAsync <ArgumentException>(async() => await index.EnsureIndexAsync(utcNow.SubtractDays(2))); existsResponse = await _client.IndexExistsAsync(index.GetIndex(utcNow.SubtractDays(2))); _logger.Trace(() => existsResponse.GetRequest()); Assert.True(existsResponse.IsValid); Assert.False(existsResponse.Exists); } } }
public async Task CanCreateDailyAliasesAsync(DateTime utcNow) { using (TestSystemClock.Install()) { SystemClock.Test.SetFixedTime(utcNow); var index = new DailyEmployeeIndex(_configuration, 1); await index.DeleteAsync(); using (new DisposableAction(() => index.DeleteAsync().GetAwaiter().GetResult())) { await index.ConfigureAsync(); var repository = new EmployeeRepository(index.Employee); for (int i = 0; i < 35; i += 5) { var employee = await repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow.SubtractDays(i))); Assert.NotNull(employee?.Id); Assert.Equal(1, await index.GetCurrentVersionAsync()); var existsResponse = await _client.IndexExistsAsync(index.GetIndex(employee.CreatedUtc)); _logger.Trace(() => existsResponse.GetRequest()); Assert.True(existsResponse.IsValid); Assert.True(existsResponse.Exists); var aliasesResponse = await _client.GetAliasAsync(a => a.Index(index.GetIndex(employee.CreatedUtc))); _logger.Trace(() => aliasesResponse.GetRequest()); Assert.True(aliasesResponse.IsValid); Assert.Equal(1, aliasesResponse.Indices.Count); var aliases = aliasesResponse.Indices.Values.Single().Select(s => s.Name).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeDailyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases)); } } } }
public async Task CanReindexTimeSeriesIndexAsync() { var version1Index = new DailyEmployeeIndex(_configuration, 1); await version1Index.DeleteAsync(); var version2Index = new DailyEmployeeIndex(_configuration, 2); await version2Index.DeleteAsync(); using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) { await version1Index.ConfigureAsync(); IEmployeeRepository version1Repository = new EmployeeRepository(version1Index); var utcNow = SystemClock.UtcNow; var employee = await version1Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow), o => o.ImmediateConsistency()); Assert.NotNull(employee?.Id); Assert.Equal(1, await version1Index.GetCurrentVersionAsync()); var aliasCountResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.Name)); _logger.LogRequest(aliasCountResponse); Assert.True(aliasCountResponse.IsValid); Assert.Equal(1, aliasCountResponse.Count); var indexCountResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.GetIndex(utcNow))); _logger.LogRequest(indexCountResponse); Assert.True(indexCountResponse.IsValid); Assert.Equal(1, indexCountResponse.Count); indexCountResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.GetVersionedIndex(utcNow, 1))); _logger.LogRequest(indexCountResponse); Assert.True(indexCountResponse.IsValid); Assert.Equal(1, indexCountResponse.Count); using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) { await version2Index.ConfigureAsync(); Assert.Equal(1, await version2Index.GetCurrentVersionAsync()); IEmployeeRepository version2Repository = new EmployeeRepository(version2Index); // Make sure we write to the old index. await version2Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow), o => o.ImmediateConsistency()); aliasCountResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.Name)); _logger.LogRequest(aliasCountResponse); Assert.True(aliasCountResponse.IsValid); Assert.Equal(2, aliasCountResponse.Count); indexCountResponse = await _client.CountAsync <Employee>(d => d.Index(version1Index.GetVersionedIndex(utcNow, 1))); _logger.LogRequest(indexCountResponse); Assert.True(indexCountResponse.IsValid); Assert.Equal(2, indexCountResponse.Count); var existsResponse = await _client.Indices.ExistsAsync(version2Index.GetVersionedIndex(utcNow, 2)); _logger.LogRequest(existsResponse); Assert.True(existsResponse.ApiCall.Success); Assert.False(existsResponse.Exists); // alias should still point to the old version until reindex var aliasesResponse = await _client.Indices.GetAliasAsync(version1Index.GetIndex(employee.CreatedUtc)); _logger.LogRequest(aliasesResponse); Assert.True(aliasesResponse.IsValid); Assert.Equal(version1Index.GetVersionedIndex(employee.CreatedUtc, 1), aliasesResponse.Indices.Single().Key); var aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeDailyAliases(version1Index, utcNow, employee.CreatedUtc), String.Join(", ", aliases)); await version2Index.ReindexAsync(); Assert.Equal(2, await version1Index.GetCurrentVersionAsync()); Assert.Equal(2, await version2Index.GetCurrentVersionAsync()); aliasesResponse = await _client.Indices.GetAliasAsync(version1Index.GetIndex(employee.CreatedUtc)); _logger.LogRequest(aliasesResponse); Assert.True(aliasesResponse.IsValid); Assert.Equal(version1Index.GetVersionedIndex(employee.CreatedUtc, 2), aliasesResponse.Indices.Single().Key); aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeDailyAliases(version1Index, utcNow, employee.CreatedUtc), String.Join(", ", aliases)); existsResponse = await _client.Indices.ExistsAsync(version1Index.GetVersionedIndex(utcNow, 1)); _logger.LogRequest(existsResponse); Assert.True(existsResponse.ApiCall.Success); Assert.False(existsResponse.Exists); existsResponse = await _client.Indices.ExistsAsync(version2Index.GetVersionedIndex(utcNow, 2)); _logger.LogRequest(existsResponse); Assert.True(existsResponse.ApiCall.Success); Assert.True(existsResponse.Exists); } } }
public async Task DailyAliasMaxAgeAsync(DateTime utcNow) { using (TestSystemClock.Install()) { SystemClock.Test.SetFixedTime(utcNow); var index = new DailyEmployeeIndex(_configuration, 1) { MaxIndexAge = TimeSpan.FromDays(45) }; await index.DeleteAsync(); using (new DisposableAction(() => index.DeleteAsync().GetAwaiter().GetResult())) { await index.ConfigureAsync(); var version1Repository = new EmployeeRepository(index.Employee); var employee = await version1Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow), o => o.ImmediateConsistency()); Assert.NotNull(employee?.Id); var existsResponse = await _client.IndexExistsAsync(index.GetIndex(employee.CreatedUtc)); _logger.Trace(() => existsResponse.GetRequest()); Assert.True(existsResponse.IsValid); Assert.True(existsResponse.Exists); var aliasesResponse = await _client.GetAliasAsync(a => a.Index(index.GetIndex(employee.CreatedUtc))); _logger.Trace(() => aliasesResponse.GetRequest()); Assert.True(aliasesResponse.IsValid); Assert.Equal(1, aliasesResponse.Indices.Count); var aliases = aliasesResponse.Indices.Values.Single().Select(s => s.Name).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeDailyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases)); employee = await version1Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow.SubtractDays(2)), o => o.ImmediateConsistency()); Assert.NotNull(employee?.Id); existsResponse = await _client.IndexExistsAsync(index.GetIndex(employee.CreatedUtc)); _logger.Trace(() => existsResponse.GetRequest()); Assert.True(existsResponse.IsValid); Assert.True(existsResponse.Exists); aliasesResponse = await _client.GetAliasAsync(a => a.Index(index.GetIndex(employee.CreatedUtc))); _logger.Trace(() => aliasesResponse.GetRequest()); Assert.True(aliasesResponse.IsValid); Assert.Equal(1, aliasesResponse.Indices.Count); aliases = aliasesResponse.Indices.Values.Single().Select(s => s.Name).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeDailyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases)); employee = await version1Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow.SubtractDays(35)), o => o.ImmediateConsistency()); Assert.NotNull(employee?.Id); existsResponse = await _client.IndexExistsAsync(index.GetIndex(employee.CreatedUtc)); _logger.Trace(() => existsResponse.GetRequest()); Assert.True(existsResponse.IsValid); Assert.True(existsResponse.Exists); aliasesResponse = await _client.GetAliasAsync(a => a.Index(index.GetIndex(employee.CreatedUtc))); _logger.Trace(() => aliasesResponse.GetRequest()); Assert.True(aliasesResponse.IsValid); Assert.Equal(1, aliasesResponse.Indices.Count); aliases = aliasesResponse.Indices.Values.Single().Select(s => s.Name).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeDailyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases)); } } }
public async Task MaintainWillCreateAliasesOnTimeSeriesIndexAsync() { using (TestSystemClock.Install()) { SystemClock.Test.SetFixedTime(SystemClock.UtcNow); var version1Index = new DailyEmployeeIndex(_configuration, 1); await version1Index.DeleteAsync(); var version2Index = new DailyEmployeeIndex(_configuration, 2); await version2Index.DeleteAsync(); // Indexes don't exist yet so the current version will be the index version. Assert.Equal(1, await version1Index.GetCurrentVersionAsync()); Assert.Equal(2, await version2Index.GetCurrentVersionAsync()); using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) { await version1Index.ConfigureAsync(); await version1Index.EnsureIndexAsync(SystemClock.UtcNow); Assert.True(_client.IndexExists(version1Index.GetVersionedIndex(SystemClock.UtcNow)).Exists); Assert.Equal(1, await version1Index.GetCurrentVersionAsync()); // delete all aliases await _configuration.Cache.RemoveAllAsync(); await DeleteAliasesAsync(version1Index.GetVersionedIndex(SystemClock.UtcNow)); using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) { await version2Index.ConfigureAsync(); await version2Index.EnsureIndexAsync(SystemClock.UtcNow); Assert.True(_client.IndexExists(version2Index.GetVersionedIndex(SystemClock.UtcNow)).Exists); Assert.Equal(2, await version2Index.GetCurrentVersionAsync()); // delete all aliases await _configuration.Cache.RemoveAllAsync(); await DeleteAliasesAsync(version2Index.GetVersionedIndex(SystemClock.UtcNow)); await _client.RefreshAsync(Indices.All); var aliasesResponse = await _client.GetAliasAsync(a => a.Index($"{version1Index.GetVersionedIndex(SystemClock.UtcNow)},{version2Index.GetVersionedIndex(SystemClock.UtcNow)}")); Assert.Equal(0, aliasesResponse.Indices.SelectMany(i => i.Value).Count()); // Indexes exist but no alias so the oldest index version will be used. Assert.Equal(1, await version1Index.GetCurrentVersionAsync()); Assert.Equal(1, await version2Index.GetCurrentVersionAsync()); await version1Index.MaintainAsync(); aliasesResponse = await _client.GetAliasAsync(a => a.Index(version1Index.GetVersionedIndex(SystemClock.UtcNow))); Assert.Equal(version1Index.Aliases.Count + 1, aliasesResponse.Indices.Single().Value.Count); aliasesResponse = await _client.GetAliasAsync(a => a.Index(version2Index.GetVersionedIndex(SystemClock.UtcNow))); Assert.Equal(0, aliasesResponse.Indices.Single().Value.Count); Assert.Equal(1, await version1Index.GetCurrentVersionAsync()); Assert.Equal(1, await version2Index.GetCurrentVersionAsync()); } } } }
public async Task MaintainDailyIndexesAsync() { using (TestSystemClock.Install()) { var index = new DailyEmployeeIndex(_configuration, 1); await index.DeleteAsync(); using (new DisposableAction(() => index.DeleteAsync().GetAwaiter().GetResult())) { await index.ConfigureAsync(); IEmployeeRepository repository = new EmployeeRepository(index); TestSystemClock.SetFrozenTime(DateTime.UtcNow.Subtract(TimeSpan.FromDays(15))); var employee = await repository.AddAsync(EmployeeGenerator.Generate(createdUtc: SystemClock.UtcNow), o => o.ImmediateConsistency()); Assert.NotNull(employee?.Id); await index.MaintainAsync(); Assert.Equal(1, await index.GetCurrentVersionAsync()); var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc)); _logger.LogRequest(existsResponse); Assert.True(existsResponse.ApiCall.Success); Assert.True(existsResponse.Exists); var aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc)); _logger.LogRequest(aliasesResponse); Assert.True(aliasesResponse.IsValid); Assert.Equal(1, aliasesResponse.Indices.Count); var aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeDailyAliases(index, SystemClock.UtcNow, employee.CreatedUtc), String.Join(", ", aliases)); TestSystemClock.SetFrozenTime(DateTime.UtcNow.Subtract(TimeSpan.FromDays(9))); index.MaxIndexAge = TimeSpan.FromDays(10); await index.MaintainAsync(); existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc)); _logger.LogRequest(existsResponse); Assert.True(existsResponse.ApiCall.Success); Assert.True(existsResponse.Exists); aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc)); _logger.LogRequest(aliasesResponse); Assert.True(aliasesResponse.IsValid); Assert.Equal(1, aliasesResponse.Indices.Count); aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeDailyAliases(index, SystemClock.UtcNow, employee.CreatedUtc), String.Join(", ", aliases)); TestSystemClock.SetFrozenTime(DateTime.UtcNow); await index.MaintainAsync(); existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc)); _logger.LogRequest(existsResponse); Assert.True(existsResponse.ApiCall.Success); Assert.False(existsResponse.Exists); aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc)); _logger.LogRequest(aliasesResponse); Assert.False(aliasesResponse.IsValid); } } }