Inheritance: Foundatio.Repositories.Elasticsearch.Configuration.VersionedIndex
コード例 #1
0
        public async Task MaintainWillCreateAliasOnVersionedIndex() {
            var version1Index = new VersionedEmployeeIndex(_configuration, 1);
            await version1Index.DeleteAsync();

            var version2Index = new VersionedEmployeeIndex(_configuration, 2);
            await version2Index.DeleteAsync();

            // Indexes don't exist yet so the current version will be the index version.
            Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
            Assert.Equal(2, await version2Index.GetCurrentVersionAsync());

            using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
                await version1Index.ConfigureAsync();
                Assert.True(_client.IndexExists(version1Index.VersionedName).Exists);
                Assert.Equal(1, await version1Index.GetCurrentVersionAsync());

                using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
                    await version2Index.ConfigureAsync();
                    Assert.True(_client.IndexExists(version2Index.VersionedName).Exists);
                    Assert.Equal(1, await version2Index.GetCurrentVersionAsync());

                    // delete all aliases
                    await _configuration.Cache.RemoveAllAsync();
                    await DeleteAliases(version1Index.VersionedName);
                    await DeleteAliases(version2Index.VersionedName);

                    await _client.RefreshAsync();
                    var aliasesResponse = await _client.GetAliasesAsync(a => a.Indices(version1Index.VersionedName, version2Index.VersionedName));
                    Assert.Equal(0, aliasesResponse.Indices.SelectMany(i => i.Value).Count());

                    // Indexes exist but no alias so the oldest index version will be used.
                    Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
                    Assert.Equal(1, await version2Index.GetCurrentVersionAsync());

                    await version1Index.MaintainAsync();
                    aliasesResponse = await _client.GetAliasesAsync(a => a.Indices(version1Index.VersionedName));
                    Assert.Equal(1, aliasesResponse.Indices.Single().Value.Count);
                    aliasesResponse = await _client.GetAliasesAsync(a => a.Indices(version2Index.VersionedName));
                    Assert.Equal(0, aliasesResponse.Indices.Single().Value.Count);

                    Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
                    Assert.Equal(1, await version2Index.GetCurrentVersionAsync());
                }
            }
        }
コード例 #2
0
        public async Task CanReindexVersionedIndexWithDeletedDocs() {
            var version1Index = new VersionedEmployeeIndex(_configuration, 1);
            await version1Index.DeleteAsync();

            var version2Index = new VersionedEmployeeIndex(_configuration, 2);
            await version2Index.DeleteAsync();

            using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
                await version1Index.ConfigureAsync();
                Assert.True(_client.IndexExists(version1Index.VersionedName).Exists);

                var repository = new EmployeeRepository(version1Index.Employee);
                var employee = await repository.AddAsync(EmployeeGenerator.Default);
                Assert.NotNull(employee?.Id);
                await _client.RefreshAsync();

                using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
                    await version2Index.ConfigureAsync();
                    Assert.True(_client.IndexExists(version2Index.VersionedName).Exists);
                    Assert.Equal(1, await version2Index.GetCurrentVersionAsync());

                    // alias should still point to the old version until reindex
                    var aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Alias(version2Index.Name));
                    Assert.True(aliasResponse.IsValid);
                    Assert.Equal(1, aliasResponse.Indices.Count);
                    Assert.Equal(version1Index.VersionedName, aliasResponse.Indices.First().Key);

                    var countdown = new AsyncCountdownEvent(1);
                    var reindexTask = version2Index.ReindexAsync((progress, message) => {
                        _logger.Info($"Reindex Progress {progress}%: {message}");
                        if (progress == 95) {
                            countdown.Signal();
                            SystemClock.Sleep(1000);
                        }

                        return Task.CompletedTask;
                    });

                    // Wait until the first reindex pass is done.
                    await countdown.WaitAsync();
                    Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
                    await repository.RemoveAllAsync();
                    await _client.RefreshAsync();

                    // Resume after everythings been indexed.
                    await reindexTask;
                    aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Alias(version2Index.Name));
                    Assert.True(aliasResponse.IsValid, aliasResponse.GetErrorMessage());
                    Assert.Equal(1, aliasResponse.Indices.Count);
                    Assert.Equal(version2Index.VersionedName, aliasResponse.Indices.First().Key);

                    Assert.Equal(2, await version1Index.GetCurrentVersionAsync());
                    Assert.Equal(2, await version2Index.GetCurrentVersionAsync());

                    var countResponse = await _client.CountAsync(d => d.Index(version1Index.VersionedName));
                    _logger.Trace(() => countResponse.GetRequest());
                    Assert.True(countResponse.ConnectionStatus.HttpStatusCode == 404, countResponse.GetErrorMessage());
                    Assert.Equal(0, countResponse.Count);

                    countResponse = await _client.CountAsync(d => d.Index(version2Index.VersionedName));
                    _logger.Trace(() => countResponse.GetRequest());
                    Assert.True(countResponse.IsValid, countResponse.GetErrorMessage());
                    Assert.Equal(1, countResponse.Count);

                    Assert.Equal(employee, await repository.GetByIdAsync(employee.Id));
                    Assert.False(_client.IndexExists(d => d.Index(version1Index.VersionedName)).Exists);
                }
            }
        }
コード例 #3
0
        public async Task CanReindexVersionedIndexWithDataInBothIndexes() {
            var version1Index = new VersionedEmployeeIndex(_configuration, 1);
            await version1Index.DeleteAsync();

            var version2Index = new VersionedEmployeeIndex(_configuration, 2);
            await version2Index.DeleteAsync();

            using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
                await version1Index.ConfigureAsync();
                Assert.True(_client.IndexExists(version1Index.VersionedName).Exists);

                var version1Repository = new EmployeeRepository(version1Index.Employee);
                var employee = await version1Repository.AddAsync(EmployeeGenerator.Default);
                Assert.NotNull(employee?.Id);
                await _client.RefreshAsync();

                using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
                    await version2Index.ConfigureAsync();
                    Assert.True(_client.IndexExists(version2Index.VersionedName).Exists);

                    // swap the alias so we write to v1 and v2 and try to reindex.
                    await _client.AliasAsync(x => x
                        .Remove(a => a.Alias(version1Index.Name).Index(version1Index.VersionedName))
                        .Add(a => a.Alias(version2Index.Name).Index(version2Index.VersionedName)));

                    var version2Repository = new EmployeeRepository(version2Index.Employee);
                    await version2Repository.AddAsync(EmployeeGenerator.Generate());
                    await _client.RefreshAsync();

                    var countResponse = await _client.CountAsync(d => d.Index(version1Index.VersionedName));
                    _logger.Trace(() => countResponse.GetRequest());
                    Assert.True(countResponse.IsValid);
                    Assert.Equal(1, countResponse.Count);

                    countResponse = await _client.CountAsync(d => d.Index(version2Index.VersionedName));
                    _logger.Trace(() => countResponse.GetRequest());
                    Assert.True(countResponse.IsValid);
                    Assert.Equal(1, countResponse.Count);

                    // swap back the alias
                    await _client.AliasAsync(x => x
                        .Remove(a => a.Alias(version2Index.Name).Index(version2Index.VersionedName))
                        .Add(a => a.Alias(version1Index.Name).Index(version1Index.VersionedName)));

                    Assert.Equal(1, await version2Index.GetCurrentVersionAsync());

                    // alias should still point to the old version until reindex
                    var aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Alias(version2Index.Name));
                    Assert.True(aliasResponse.IsValid);
                    Assert.Equal(1, aliasResponse.Indices.Count);
                    Assert.Equal(version1Index.VersionedName, aliasResponse.Indices.First().Key);

                    await version2Index.ReindexAsync();

                    aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Alias(version2Index.Name));
                    Assert.True(aliasResponse.IsValid);
                    Assert.Equal(1, aliasResponse.Indices.Count);
                    Assert.Equal(version2Index.VersionedName, aliasResponse.Indices.First().Key);

                    Assert.Equal(2, await version1Index.GetCurrentVersionAsync());
                    Assert.Equal(2, await version2Index.GetCurrentVersionAsync());

                    countResponse = await _client.CountAsync(d => d.Index(version2Index.VersionedName));
                    _logger.Trace(() => countResponse.GetRequest());
                    Assert.True(countResponse.IsValid);
                    Assert.Equal(2, countResponse.Count);

                    Assert.False(_client.IndexExists(d => d.Index(version1Index.VersionedName)).Exists);
                }
            }
        }
コード例 #4
0
        public async Task CanReindexVersionedIndexWithCorrectMappings() {
            var version1Index = new VersionedEmployeeIndex(_configuration, 1);
            await version1Index.DeleteAsync();

            var version2Index = new VersionedEmployeeIndex(_configuration, 2);
            version2Index.DiscardIndexesOnReindex = false;
            await version2Index.DeleteAsync();

            using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
                await version1Index.ConfigureAsync();
                var version1Repository = new EmployeeRepository(version1Index.Employee);

                var utcNow = SystemClock.UtcNow;
                var employee = await version1Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow));
                Assert.NotNull(employee?.Id);
                await _client.RefreshAsync();

                using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
                    await version2Index.ConfigureAsync();

                    await version2Index.ReindexAsync();

                    var existsResponse = await _client.IndexExistsAsync(d => d.Index(version1Index.VersionedName));
                    _logger.Trace(() => existsResponse.GetRequest());
                    Assert.True(existsResponse.IsValid);
                    Assert.True(existsResponse.Exists);

                    var mappingResponse = await _client.GetMappingAsync<Employee>(m => m.Index(version1Index.VersionedName));
                    _logger.Trace(() => mappingResponse.GetRequest());
                    Assert.True(mappingResponse.IsValid);
                    Assert.NotNull(mappingResponse.Mappings);

                    existsResponse = await _client.IndexExistsAsync(d => d.Index(version2Index.VersionedName));
                    _logger.Trace(() => existsResponse.GetRequest());
                    Assert.True(existsResponse.IsValid);
                    Assert.True(existsResponse.Exists);

                    string version1Mappings = ToJson(mappingResponse.Mappings);
                    mappingResponse = await _client.GetMappingAsync<Employee>(m => m.Index(version1Index.VersionedName));
                    _logger.Trace(() => mappingResponse.GetRequest());
                    Assert.True(mappingResponse.IsValid);
                    Assert.NotNull(mappingResponse.Mappings);
                    Assert.Equal(version1Mappings, ToJson(mappingResponse.Mappings).Replace("-v2", "-v1"));
                }
            }
        }
コード例 #5
0
        public async Task CanReindexVersionedIndex() {
            var version1Index = new VersionedEmployeeIndex(_configuration, 1);
            await version1Index.DeleteAsync();

            var version2Index = new VersionedEmployeeIndex(_configuration, 2);
            await version2Index.DeleteAsync();

            using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
                await version1Index.ConfigureAsync();
                Assert.True(_client.IndexExists(version1Index.VersionedName).Exists);

                var indexes = _client.GetIndicesPointingToAlias(version1Index.Name);
                Assert.Equal(1, indexes.Count);

                var aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Alias(version1Index.Name));
                _logger.Trace(() => aliasResponse.GetRequest());
                Assert.True(aliasResponse.IsValid);
                Assert.Equal(1, aliasResponse.Indices.Count);
                Assert.Equal(version1Index.VersionedName, aliasResponse.Indices.First().Key);

                var version1Repository = new EmployeeRepository(version1Index.Employee);
                var employee = await version1Repository.AddAsync(EmployeeGenerator.Default);
                await _client.RefreshAsync();
                Assert.NotNull(employee?.Id);

                var countResponse = await _client.CountAsync(d => d.Index(version1Index.Name));
                _logger.Trace(() => countResponse.GetRequest());
                Assert.True(countResponse.IsValid);
                Assert.Equal(1, countResponse.Count);

                Assert.Equal(1, await version1Index.GetCurrentVersionAsync());

                using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
                    await version2Index.ConfigureAsync();
                    Assert.True(_client.IndexExists(version2Index.VersionedName).Exists);

                    // Make sure we can write to the index still. Should go to the old index until after the reindex is complete.
                    var version2Repository = new EmployeeRepository(version2Index.Employee);
                    await version2Repository.AddAsync(EmployeeGenerator.Generate());
                    await _client.RefreshAsync();

                    countResponse = await _client.CountAsync(d => d.Index(version1Index.VersionedName));
                    _logger.Trace(() => countResponse.GetRequest());
                    Assert.True(countResponse.IsValid);
                    Assert.Equal(2, countResponse.Count);

                    countResponse = await _client.CountAsync(d => d.Index(version2Index.VersionedName));
                    _logger.Trace(() => countResponse.GetRequest());
                    Assert.True(countResponse.IsValid);
                    Assert.Equal(0, countResponse.Count);

                    Assert.Equal(1, await version2Index.GetCurrentVersionAsync());

                    // alias should still point to the old version until reindex
                    aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Alias(version2Index.Name));
                    Assert.True(aliasResponse.IsValid);
                    Assert.Equal(1, aliasResponse.Indices.Count);
                    Assert.Equal(version1Index.VersionedName, aliasResponse.Indices.First().Key);

                    await version2Index.ReindexAsync();

                    aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Alias(version2Index.Name));
                    Assert.True(aliasResponse.IsValid);
                    Assert.Equal(1, aliasResponse.Indices.Count);
                    Assert.Equal(version2Index.VersionedName, aliasResponse.Indices.First().Key);

                    Assert.Equal(2, await version1Index.GetCurrentVersionAsync());
                    Assert.Equal(2, await version2Index.GetCurrentVersionAsync());

                    countResponse = await _client.CountAsync(d => d.Index(version2Index.VersionedName));
                    _logger.Trace(() => countResponse.GetRequest());
                    Assert.True(countResponse.IsValid);
                    Assert.Equal(2, countResponse.Count);

                    Assert.False(_client.IndexExists(d => d.Index(version1Index.VersionedName)).Exists);

                    employee = await version2Repository.AddAsync(EmployeeGenerator.Default);
                    await _client.RefreshAsync();
                    Assert.NotNull(employee?.Id);

                    countResponse = await _client.CountAsync(d => d.Index(version2Index.Name));
                    _logger.Trace(() => countResponse.GetRequest());
                    Assert.True(countResponse.IsValid);
                    Assert.Equal(3, countResponse.Count);
                }
            }
        }
コード例 #6
0
        public async Task CanResumeReindex() {
            const int numberOfEmployeesToCreate = 2000;

            var version1Index = new VersionedEmployeeIndex(_configuration, 1);
            await version1Index.DeleteAsync();

            var version2Index = new VersionedEmployeeIndex(_configuration, 2);
            await version2Index.DeleteAsync();

            using (new DisposableAction(() => version1Index.DeleteAsync().GetAwaiter().GetResult())) {
                await version1Index.ConfigureAsync();
                Assert.True(_client.IndexExists(version1Index.VersionedName).Exists);

                var version1Repository = new EmployeeRepository(version1Index.Employee);
                await version1Repository.AddAsync(EmployeeGenerator.GenerateEmployees(numberOfEmployeesToCreate));
                await _client.RefreshAsync();

                var countResponse = await _client.CountAsync(d => d.Index(version1Index.Name));
                _logger.Trace(() => countResponse.GetRequest());
                Assert.True(countResponse.IsValid);
                Assert.Equal(numberOfEmployeesToCreate, countResponse.Count);

                Assert.Equal(1, await version1Index.GetCurrentVersionAsync());

                using (new DisposableAction(() => version2Index.DeleteAsync().GetAwaiter().GetResult())) {
                    await version2Index.ConfigureAsync();
                    Assert.True(_client.IndexExists(version2Index.VersionedName).Exists);

                    await Assert.ThrowsAsync<ApplicationException>(async () => await version2Index.ReindexAsync((progress, message) => {
                        _logger.Info("Reindex Progress {0}%: {1}", progress, message);
                        // TODO: Need to make this so it happens randomly in the middle of a batch
                        if (progress >= 45)
                            throw new ApplicationException("Random Error");

                        return Task.CompletedTask;
                    }));

                    Assert.Equal(1, await version1Index.GetCurrentVersionAsync());
                    await version2Index.ReindexAsync();

                    var aliasResponse = await _client.GetAliasAsync(descriptor => descriptor.Alias(version2Index.Name));
                    Assert.True(aliasResponse.IsValid);
                    Assert.Equal(1, aliasResponse.Indices.Count);
                    Assert.Equal(version2Index.VersionedName, aliasResponse.Indices.First().Key);

                    Assert.Equal(2, await version1Index.GetCurrentVersionAsync());
                    Assert.Equal(2, await version2Index.GetCurrentVersionAsync());

                    countResponse = await _client.CountAsync(d => d.Index(version2Index.VersionedName));
                    _logger.Trace(() => countResponse.GetRequest());
                    Assert.True(countResponse.IsValid);
                    Assert.Equal(numberOfEmployeesToCreate, countResponse.Count);

                    Assert.False(_client.IndexExists(d => d.Index(version1Index.VersionedName)).Exists);
                }
            }
        }