public virtual void ConfigureIndexes(IElasticClient client, IEnumerable<IElasticIndex> indexes = null) { if (indexes == null) indexes = GetIndexes(); foreach (var idx in indexes) { int currentVersion = GetAliasVersion(client, idx.AliasName); IIndicesOperationResponse response = null; var templatedIndex = idx as ITemplatedElasticIndex; if (templatedIndex != null) response = client.PutTemplate(idx.VersionedName, template => templatedIndex.CreateTemplate(template).AddAlias(idx.AliasName)); else if (!client.IndexExists(idx.VersionedName).Exists) response = client.CreateIndex(idx.VersionedName, descriptor => idx.CreateIndex(descriptor).AddAlias(idx.AliasName)); Debug.Assert(response == null || response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the index or template."); // Add existing indexes to the alias. if (!client.AliasExists(idx.AliasName).Exists) { if (templatedIndex != null) { var indices = client.IndicesStats().Indices.Where(kvp => kvp.Key.StartsWith(idx.VersionedName)).Select(kvp => kvp.Key).ToList(); if (indices.Count > 0) { var descriptor = new AliasDescriptor(); foreach (string name in indices) descriptor.Add(add => add.Index(name).Alias(idx.AliasName)); response = client.Alias(descriptor); } } else { response = client.Alias(a => a.Add(add => add.Index(idx.VersionedName).Alias(idx.AliasName))); } Debug.Assert(response != null && response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the alias."); } // already on current version if (currentVersion >= idx.Version || currentVersion < 1) continue; var reindexWorkItem = new ReindexWorkItem { OldIndex = String.Concat(idx.AliasName, "-v", currentVersion), NewIndex = idx.VersionedName, Alias = idx.AliasName, DeleteOld = true, ParentMaps = idx.GetIndexTypes() .Select(kvp => new ParentMap {Type = kvp.Value.Name, ParentPath = kvp.Value.ParentPath}) .Where(m => !String.IsNullOrEmpty(m.ParentPath)) .ToList() }; bool isReindexing = _lockProvider.IsLockedAsync(String.Concat("reindex:", reindexWorkItem.Alias, reindexWorkItem.OldIndex, reindexWorkItem.NewIndex)).Result; // already reindexing if (isReindexing) continue; // enqueue reindex to new version _lockProvider.TryUsingAsync("enqueue-reindex", () => _workItemQueue.EnqueueAsync(reindexWorkItem), TimeSpan.Zero, CancellationToken.None).Wait(); } }
public IBulkAliasResponse Alias(string aliasName, string indexName, string callerName = "") { Func <BulkAliasDescriptor, IBulkAliasRequest> selector = a => a.Add(add => add.Index(indexName).Alias(aliasName)); var timer = Stopwatch.StartNew(); var result = _client.Alias(selector); SendLog(result.ApiCall, null, timer.ElapsedMilliseconds, $"Alias {aliasName} > {indexName}"); return(result); }
public void Initialize() { var uri = _options.ElasticsearchServerAddress; _logger.LogInformation("Initialing elastic search with uri: " + uri); var pool = new SingleNodeConnectionPool(new Uri(uri)); var connectionString = new ConnectionSettings( pool, new HttpConnection(), new SerializerFactory(s => new GeoJsonNetSerializer(s, _geometryFactory))) .PrettyJson(); _elasticClient = new ElasticClient(connectionString); if (_elasticClient.IndexExists(OSM_POIS_INDEX1).Exists == false && _elasticClient.IndexExists(OSM_POIS_INDEX2).Exists == false) { CreatePointsOfInterestIndex(OSM_POIS_INDEX1); _elasticClient.Alias(a => a.Add(add => add.Alias(OSM_POIS_ALIAS).Index(OSM_POIS_INDEX1))); } if (_elasticClient.IndexExists(OSM_POIS_INDEX1).Exists&& _elasticClient.IndexExists(OSM_POIS_INDEX2).Exists) { _elasticClient.DeleteIndex(OSM_POIS_INDEX2); } if (_elasticClient.IndexExists(OSM_HIGHWAYS_INDEX1).Exists == false && _elasticClient.IndexExists(OSM_HIGHWAYS_INDEX2).Exists == false) { CreateHighwaysIndex(OSM_HIGHWAYS_INDEX1); _elasticClient.Alias(a => a.Add(add => add.Alias(OSM_HIGHWAYS_ALIAS).Index(OSM_HIGHWAYS_INDEX1))); } if (_elasticClient.IndexExists(OSM_HIGHWAYS_INDEX1).Exists&& _elasticClient.IndexExists(OSM_HIGHWAYS_INDEX2).Exists) { _elasticClient.DeleteIndex(OSM_HIGHWAYS_INDEX2); } if (_elasticClient.IndexExists(SHARES).Exists == false) { _elasticClient.CreateIndex(SHARES); } if (_elasticClient.IndexExists(CUSTOM_USER_LAYERS).Exists == false) { _elasticClient.CreateIndex(CUSTOM_USER_LAYERS); } if (_elasticClient.IndexExists(IMAGES).Exists == false) { CreateImagesIndex(); } _logger.LogInformation("Finished initialing elasticsearch with uri: " + uri); }
public void ReIndex() { const string newIndex = "cx_demo_claims_2"; var reindex = _client.Reindex <ClaimModel>(Storage.ClaimIndex, newIndex, (inbox => inbox .MatchAll())); var observer = new ReindexObserver(onNext: response => { Console.WriteLine($"Indexing documents"); }, onError: error => { Console.WriteLine($"Document error occured: {error.Message}"); }, onCompleted: () => { Console.WriteLine($"Document re-index"); }); reindex.Subscribe(observer); _client.DeleteIndex(Indices.Index(new IndexName { Name = Storage.ClaimIndex })); _client.Alias(x => x.Add(a => a.Alias(Storage.ClaimsAlias).Index(newIndex))); }
public void Initialize(string uri = "http://localhost:9200/") { _logger.LogInformation("Initialing elastic search with uri: " + uri); var pool = new SingleNodeConnectionPool(new Uri(uri)); var connectionString = new ConnectionSettings( pool, new HttpConnection(), new SerializerFactory(s => new GeoJsonNetSerializer(s))) .PrettyJson(); _elasticClient = new ElasticClient(connectionString); if (_elasticClient.IndexExists(OSM_POIS_INDEX1).Exists == false && _elasticClient.IndexExists(OSM_POIS_INDEX2).Exists == false) { CreatePointsOfInterestIndex(OSM_POIS_INDEX1); _elasticClient.Alias(a => a.Add(add => add.Alias(OSM_POIS_ALIAS).Index(OSM_POIS_INDEX1))); } if (_elasticClient.IndexExists(OSM_POIS_INDEX1).Exists&& _elasticClient.IndexExists(OSM_POIS_INDEX2).Exists) { _elasticClient.DeleteIndex(OSM_POIS_INDEX2); } if (_elasticClient.IndexExists(OSM_HIGHWAYS_INDEX1).Exists == false && _elasticClient.IndexExists(OSM_HIGHWAYS_INDEX2).Exists == false) { CreateHighwaysIndex(OSM_HIGHWAYS_INDEX1); _elasticClient.Alias(a => a.Add(add => add.Alias(OSM_HIGHWAYS_ALIAS).Index(OSM_HIGHWAYS_INDEX1))); } if (_elasticClient.IndexExists(OSM_HIGHWAYS_INDEX1).Exists&& _elasticClient.IndexExists(OSM_HIGHWAYS_INDEX2).Exists) { _elasticClient.DeleteIndex(OSM_HIGHWAYS_INDEX2); } if (_elasticClient.IndexExists(RATINGS).Exists == false) { _elasticClient.CreateIndex(RATINGS); } if (_elasticClient.IndexExists(SHARES).Exists == false) { _elasticClient.CreateIndex(SHARES); } if (_elasticClient.IndexExists(USER_LAYERS).Exists == false) { _elasticClient.CreateIndex(SHARES); } _logger.LogInformation("Finished initialing elasticsearch with uri: " + uri); }
public void ConfigureIndexes(IElasticClient client) { foreach (var index in GetIndexes()) { IIndicesOperationResponse response = null; int currentVersion = GetAliasVersion(client, index.Name); var templatedIndex = index as ITemplatedElasticSeachIndex; if (templatedIndex != null) response = client.PutTemplate(index.VersionedName, template => templatedIndex.CreateTemplate(template).AddAlias(index.Name)); else if (!client.IndexExists(index.VersionedName).Exists) response = client.CreateIndex(index.VersionedName, descriptor => index.CreateIndex(descriptor).AddAlias(index.Name)); Debug.Assert(response == null || response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the index or template."); // Add existing indexes to the alias. if (!client.AliasExists(index.Name).Exists) { if (templatedIndex != null) { var indices = client.IndicesStats().Indices.Where(kvp => kvp.Key.StartsWith(index.VersionedName)).Select(kvp => kvp.Key).ToList(); if (indices.Count > 0) { var descriptor = new AliasDescriptor(); foreach (string name in indices) descriptor.Add(add => add.Index(name).Alias(index.Name)); response = client.Alias(descriptor); } } else { response = client.Alias(a => a.Add(add => add.Index(index.VersionedName).Alias(index.Name))); } Debug.Assert(response != null && response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the alias."); } // already on current version if (currentVersion >= index.Version || currentVersion < 1) continue; // upgrade _lockProvider.TryUsingAsync("reindex", async () => { await _workItemQueue.EnqueueAsync(new ReindexWorkItem { OldIndex = String.Concat(index.Name, "-v", currentVersion), NewIndex = index.VersionedName, Alias = index.Name, DeleteOld = true }); }, TimeSpan.Zero, CancellationToken.None); } }
public void Initialize(string uri = "http://localhost:9200/") { _logger.LogInformation("Initialing elastic search with uri: " + uri); var pool = new SingleNodeConnectionPool(new Uri(uri)); var connectionString = new ConnectionSettings( pool, new HttpConnection(), new ConnectionSettings.SourceSerializerFactory((builtin, settings) => new JsonNetSerializer(builtin, settings, () => new Newtonsoft.Json.JsonSerializerSettings { Converters = GeoJsonSerializer.Create(_geometryFactory, 3).Converters }))) .PrettyJson(); _elasticClient = new ElasticClient(connectionString); if (_elasticClient.IndexExists(OSM_POIS_INDEX1).Exists == false && _elasticClient.IndexExists(OSM_POIS_INDEX2).Exists == false) { CreatePointsOfInterestIndex(OSM_POIS_INDEX1); _elasticClient.Alias(a => a.Add(add => add.Alias(OSM_POIS_ALIAS).Index(OSM_POIS_INDEX1))); } if (_elasticClient.IndexExists(OSM_POIS_INDEX1).Exists&& _elasticClient.IndexExists(OSM_POIS_INDEX2).Exists) { _elasticClient.DeleteIndex(OSM_POIS_INDEX2); } if (_elasticClient.IndexExists(OSM_HIGHWAYS_INDEX1).Exists == false && _elasticClient.IndexExists(OSM_HIGHWAYS_INDEX2).Exists == false) { CreateHighwaysIndex(OSM_HIGHWAYS_INDEX1); _elasticClient.Alias(a => a.Add(add => add.Alias(OSM_HIGHWAYS_ALIAS).Index(OSM_HIGHWAYS_INDEX1))); } if (_elasticClient.IndexExists(OSM_HIGHWAYS_INDEX1).Exists&& _elasticClient.IndexExists(OSM_HIGHWAYS_INDEX2).Exists) { _elasticClient.DeleteIndex(OSM_HIGHWAYS_INDEX2); } if (_elasticClient.IndexExists(SHARES).Exists == false) { _elasticClient.CreateIndex(SHARES); } if (_elasticClient.IndexExists(CUSTOM_USER_LAYERS).Exists == false) { _elasticClient.CreateIndex(CUSTOM_USER_LAYERS); } _logger.LogInformation("Finished initialing elasticsearch with uri: " + uri); }
/// <summary> /// 重建索引 /// </summary> /// <param name="client"></param> /// <param name="oldIndexName">就索引名字</param> /// <param name="newIndexName">新索引名字</param> /// <param name="shardsNumber"></param> /// <returns></returns> public static bool ReIndex <T>(IElasticClient client, string oldIndexName, string newIndexName, int shardsNumber = 1) where T : class { //检查新索引别名 var alias = client.AliasExists($"{newIndexName}-reindex"); //如果改别名不存在 if (!alias.Exists) { //检查是否存在新索引 IExistsResponse newIndex = client.IndexExists(newIndexName); client.UpdateIndexSettings(Indices.Index(newIndexName), setting => setting.IndexSettings(isetting => isetting.Setting("max_result_window", 100000))); //如果新索引不存在则创建一个新索引,需要修改各项创建索引时才需要的参数则在这处理 if (!newIndex.Exists) { CreateIndexDescriptor createIndex = new CreateIndexDescriptor(newIndexName) .Settings(s => s.NumberOfShards(shardsNumber).Analysis(a => a.Analyzers(aa => aa.Language("standard_listing", sa => sa.Language(Language.Chinese))))) .Mappings(ms => ms.Map <T>(m => m.AutoMap())); ICreateIndexResponse create = client.CreateIndex(createIndex); Console.WriteLine($"{newIndexName}索引创建:{create.ApiCall.Success}"); //给新创建的索引添加一个索引别名表示时重新创建的 var r2 = client.Alias(t => t.Add(a => a.Index(newIndexName).Alias($"{newIndexName}-reindex"))); Console.WriteLine($"索引别名{newIndexName}=>{newIndexName}:{r2.ApiCall.Success}"); } //将老索引中的文档索引导新建的索引 var r1 = client.ReindexOnServer(r => r.Source(s => s.Index(Indices.Index(oldIndexName))).Destination(d => d.Index(newIndexName))); Console.WriteLine($"重新索引:{r1.ApiCall.Success}"); if (r1.ApiCall.Success) { //老索引中的所有文档全部到新索引之后删除老索引 client.DeleteIndex(Indices.Index(oldIndexName)); //新索引设置别名为老索引的名字 var r3 = client.Alias(t => t.Add(a => a.Index(newIndexName).Alias(oldIndexName))); Console.WriteLine($"索引别名{newIndexName}=>{oldIndexName}:{r3.ApiCall.Success}"); } } return(alias.Exists); }
public void SwitchAliasToIndex(string indexName) { var catAliases = _client.CatAliases(new CatAliasesRequest(_aliasName)); var oldIndexes = catAliases.Records.Select(x => x.Index); var response = _client.Alias(a => RemoveIndexesFromAlias(a, oldIndexes).Add(i => i.Index(indexName).Alias(_aliasName))); if (!response.IsValid) { throw new Exception("Failed while switching index alias.", response.OriginalException); } foreach (var index in oldIndexes) { _client.DeleteIndexAsync(new DeleteIndexRequest(index)); } }
public bool MakeIndexCurrentAlias(string indexName) { //Get the Indices assigned to an Alias string[] indicesToRemove = GetIndicesForAlias(); //So the alias command is a bulk operation and //is not done on a single index or alias, but //it is a series of add/remove commands of which //require an alias name and an index. So we need //to do a bit more with our command descriptor than //we usually do. Always remember that for the compiler //that // command(x => x.Param()); // is the same as // command(x => { x.Param(); return x }); var response = _client.Alias(a => { //Add the new index a.Add(add => add .Index(indexName) .Alias(this._config.AliasName) ); //Remove any old ones foreach (string oldIndex in indicesToRemove) { a.Remove(del => del .Index(oldIndex) .Alias(this._config.AliasName) ); } return(a); }); if (!response.IsValid) { throw new Exception("Error swapping indices for alias: " + this._config.AliasName, response.OriginalException); } return(true); }
public void ConfigureIndexes(IElasticClient client) { foreach (var index in GetIndexes()) { IIndicesOperationResponse response = null; int currentVersion = GetAliasVersion(client, index.Name); var templatedIndex = index as ITemplatedElasticSeachIndex; if (templatedIndex != null) { response = client.PutTemplate(index.VersionedName, template => templatedIndex.CreateTemplate(template).AddAlias(index.Name)); } else if (!client.IndexExists(index.VersionedName).Exists) { response = client.CreateIndex(index.VersionedName, descriptor => index.CreateIndex(descriptor).AddAlias(index.Name)); } Debug.Assert(response == null || response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the index or template."); // Add existing indexes to the alias. if (!client.AliasExists(index.Name).Exists) { if (templatedIndex != null) { var indices = client.IndicesStats().Indices.Where(kvp => kvp.Key.StartsWith(index.VersionedName)).Select(kvp => kvp.Key).ToList(); if (indices.Count > 0) { var descriptor = new AliasDescriptor(); foreach (string name in indices) { descriptor.Add(add => add.Index(name).Alias(index.Name)); } response = client.Alias(descriptor); } } else { response = client.Alias(a => a.Add(add => add.Index(index.VersionedName).Alias(index.Name))); } Debug.Assert(response != null && response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the alias."); } // already on current version if (currentVersion >= index.Version || currentVersion < 1) { continue; } // upgrade _lockProvider.TryUsingAsync("reindex", async() => { await _workItemQueue.EnqueueAsync(new ReindexWorkItem { OldIndex = String.Concat(index.Name, "-v", currentVersion), NewIndex = index.VersionedName, Alias = index.Name, DeleteOld = true }); }, TimeSpan.Zero, CancellationToken.None); } }
public virtual void ConfigureIndexes(IElasticClient client, IEnumerable <IElasticIndex> indexes = null) { if (indexes == null) { indexes = GetIndexes(); } foreach (var idx in indexes) { int currentVersion = GetAliasVersion(client, idx.AliasName); IIndicesOperationResponse response = null; var templatedIndex = idx as ITemplatedElasticIndex; if (templatedIndex != null) { response = client.PutTemplate(idx.VersionedName, template => templatedIndex.CreateTemplate(template).AddAlias(idx.AliasName)); } else if (!client.IndexExists(idx.VersionedName).Exists) { response = client.CreateIndex(idx.VersionedName, descriptor => idx.CreateIndex(descriptor).AddAlias(idx.AliasName)); } Debug.Assert(response == null || response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the index or template."); // Add existing indexes to the alias. if (!client.AliasExists(idx.AliasName).Exists) { if (templatedIndex != null) { var indices = client.IndicesStats().Indices.Where(kvp => kvp.Key.StartsWith(idx.VersionedName)).Select(kvp => kvp.Key).ToList(); if (indices.Count > 0) { var descriptor = new AliasDescriptor(); foreach (string name in indices) { descriptor.Add(add => add.Index(name).Alias(idx.AliasName)); } response = client.Alias(descriptor); } } else { response = client.Alias(a => a.Add(add => add.Index(idx.VersionedName).Alias(idx.AliasName))); } Debug.Assert(response != null && response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the alias."); } // already on current version if (currentVersion >= idx.Version || currentVersion < 1) { continue; } var reindexWorkItem = new ReindexWorkItem { OldIndex = String.Concat(idx.AliasName, "-v", currentVersion), NewIndex = idx.VersionedName, Alias = idx.AliasName, DeleteOld = true, ParentMaps = idx.GetIndexTypes() .Select(kvp => new ParentMap { Type = kvp.Value.Name, ParentPath = kvp.Value.ParentPath }) .Where(m => !String.IsNullOrEmpty(m.ParentPath)) .ToList() }; bool isReindexing = _lockProvider.IsLockedAsync(String.Concat("reindex:", reindexWorkItem.Alias, reindexWorkItem.OldIndex, reindexWorkItem.NewIndex)).Result; // already reindexing if (isReindexing) { continue; } // enqueue reindex to new version _lockProvider.TryUsingAsync("enqueue-reindex", () => _workItemQueue.EnqueueAsync(reindexWorkItem), TimeSpan.Zero, CancellationToken.None).Wait(); } }
public void OnSuccess(IElasticClient client, IIndicesOperationResponse response) { client.Alias(a => a.Add(aa => aa.Alias(client.Infer.DefaultIndex).Index(_indexName))); }