public virtual void ConfigureIndexes(IElasticClient client, IEnumerable<IElasticIndex> indexes = null) {
if (indexes == null)
indexes = GetIndexes();
foreach (var idx in indexes) {
int currentVersion = GetAliasVersion(client, idx.AliasName);
IIndicesOperationResponse response = null;
var templatedIndex = idx as ITemplatedElasticIndex;
if (templatedIndex != null)
response = client.PutTemplate(idx.VersionedName, template => templatedIndex.CreateTemplate(template).AddAlias(idx.AliasName));
else if (!client.IndexExists(idx.VersionedName).Exists)
response = client.CreateIndex(idx.VersionedName, descriptor => idx.CreateIndex(descriptor).AddAlias(idx.AliasName));
Debug.Assert(response == null || response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the index or template.");
// Add existing indexes to the alias.
if (!client.AliasExists(idx.AliasName).Exists) {
if (templatedIndex != null) {
var indices = client.IndicesStats().Indices.Where(kvp => kvp.Key.StartsWith(idx.VersionedName)).Select(kvp => kvp.Key).ToList();
if (indices.Count > 0) {
var descriptor = new AliasDescriptor();
foreach (string name in indices)
descriptor.Add(add => add.Index(name).Alias(idx.AliasName));
response = client.Alias(descriptor);
}
} else {
response = client.Alias(a => a.Add(add => add.Index(idx.VersionedName).Alias(idx.AliasName)));
}
Debug.Assert(response != null && response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the alias.");
}
// already on current version
if (currentVersion >= idx.Version || currentVersion < 1)
continue;
var reindexWorkItem = new ReindexWorkItem {
OldIndex = String.Concat(idx.AliasName, "-v", currentVersion),
NewIndex = idx.VersionedName,
Alias = idx.AliasName,
DeleteOld = true,
ParentMaps = idx.GetIndexTypes()
.Select(kvp => new ParentMap {Type = kvp.Value.Name, ParentPath = kvp.Value.ParentPath})
.Where(m => !String.IsNullOrEmpty(m.ParentPath))
.ToList()
};
bool isReindexing = _lockProvider.IsLockedAsync(String.Concat("reindex:", reindexWorkItem.Alias, reindexWorkItem.OldIndex, reindexWorkItem.NewIndex)).Result;
// already reindexing
if (isReindexing)
continue;
// enqueue reindex to new version
_lockProvider.TryUsingAsync("enqueue-reindex", () => _workItemQueue.EnqueueAsync(reindexWorkItem), TimeSpan.Zero, CancellationToken.None).Wait();
}
}
public IBulkAliasResponse Alias(string aliasName, string indexName, string callerName = "")
{
Func <BulkAliasDescriptor, IBulkAliasRequest> selector = a => a.Add(add => add.Index(indexName).Alias(aliasName));
var timer = Stopwatch.StartNew();
var result = _client.Alias(selector);
SendLog(result.ApiCall, null, timer.ElapsedMilliseconds, $"Alias {aliasName} > {indexName}");
return(result);
}
public void Initialize()
{
var uri = _options.ElasticsearchServerAddress;
_logger.LogInformation("Initialing elastic search with uri: " + uri);
var pool = new SingleNodeConnectionPool(new Uri(uri));
var connectionString = new ConnectionSettings(
pool,
new HttpConnection(),
new SerializerFactory(s => new GeoJsonNetSerializer(s, _geometryFactory)))
.PrettyJson();
_elasticClient = new ElasticClient(connectionString);
if (_elasticClient.IndexExists(OSM_POIS_INDEX1).Exists == false &&
_elasticClient.IndexExists(OSM_POIS_INDEX2).Exists == false)
{
CreatePointsOfInterestIndex(OSM_POIS_INDEX1);
_elasticClient.Alias(a => a.Add(add => add.Alias(OSM_POIS_ALIAS).Index(OSM_POIS_INDEX1)));
}
if (_elasticClient.IndexExists(OSM_POIS_INDEX1).Exists&&
_elasticClient.IndexExists(OSM_POIS_INDEX2).Exists)
{
_elasticClient.DeleteIndex(OSM_POIS_INDEX2);
}
if (_elasticClient.IndexExists(OSM_HIGHWAYS_INDEX1).Exists == false &&
_elasticClient.IndexExists(OSM_HIGHWAYS_INDEX2).Exists == false)
{
CreateHighwaysIndex(OSM_HIGHWAYS_INDEX1);
_elasticClient.Alias(a => a.Add(add => add.Alias(OSM_HIGHWAYS_ALIAS).Index(OSM_HIGHWAYS_INDEX1)));
}
if (_elasticClient.IndexExists(OSM_HIGHWAYS_INDEX1).Exists&&
_elasticClient.IndexExists(OSM_HIGHWAYS_INDEX2).Exists)
{
_elasticClient.DeleteIndex(OSM_HIGHWAYS_INDEX2);
}
if (_elasticClient.IndexExists(SHARES).Exists == false)
{
_elasticClient.CreateIndex(SHARES);
}
if (_elasticClient.IndexExists(CUSTOM_USER_LAYERS).Exists == false)
{
_elasticClient.CreateIndex(CUSTOM_USER_LAYERS);
}
if (_elasticClient.IndexExists(IMAGES).Exists == false)
{
CreateImagesIndex();
}
_logger.LogInformation("Finished initialing elasticsearch with uri: " + uri);
}
public void ReIndex()
{
const string newIndex = "cx_demo_claims_2";
var reindex =
_client.Reindex <ClaimModel>(Storage.ClaimIndex, newIndex, (inbox => inbox
.MatchAll()));
var observer = new ReindexObserver(onNext: response =>
{
Console.WriteLine($"Indexing documents");
}, onError: error =>
{
Console.WriteLine($"Document error occured: {error.Message}");
},
onCompleted: () =>
{
Console.WriteLine($"Document re-index");
});
reindex.Subscribe(observer);
_client.DeleteIndex(Indices.Index(new IndexName {
Name = Storage.ClaimIndex
}));
_client.Alias(x => x.Add(a => a.Alias(Storage.ClaimsAlias).Index(newIndex)));
}
public void Initialize(string uri = "http://localhost:9200/")
{
_logger.LogInformation("Initialing elastic search with uri: " + uri);
var pool = new SingleNodeConnectionPool(new Uri(uri));
var connectionString = new ConnectionSettings(
pool,
new HttpConnection(),
new SerializerFactory(s => new GeoJsonNetSerializer(s)))
.PrettyJson();
_elasticClient = new ElasticClient(connectionString);
if (_elasticClient.IndexExists(OSM_POIS_INDEX1).Exists == false &&
_elasticClient.IndexExists(OSM_POIS_INDEX2).Exists == false)
{
CreatePointsOfInterestIndex(OSM_POIS_INDEX1);
_elasticClient.Alias(a => a.Add(add => add.Alias(OSM_POIS_ALIAS).Index(OSM_POIS_INDEX1)));
}
if (_elasticClient.IndexExists(OSM_POIS_INDEX1).Exists&&
_elasticClient.IndexExists(OSM_POIS_INDEX2).Exists)
{
_elasticClient.DeleteIndex(OSM_POIS_INDEX2);
}
if (_elasticClient.IndexExists(OSM_HIGHWAYS_INDEX1).Exists == false &&
_elasticClient.IndexExists(OSM_HIGHWAYS_INDEX2).Exists == false)
{
CreateHighwaysIndex(OSM_HIGHWAYS_INDEX1);
_elasticClient.Alias(a => a.Add(add => add.Alias(OSM_HIGHWAYS_ALIAS).Index(OSM_HIGHWAYS_INDEX1)));
}
if (_elasticClient.IndexExists(OSM_HIGHWAYS_INDEX1).Exists&&
_elasticClient.IndexExists(OSM_HIGHWAYS_INDEX2).Exists)
{
_elasticClient.DeleteIndex(OSM_HIGHWAYS_INDEX2);
}
if (_elasticClient.IndexExists(RATINGS).Exists == false)
{
_elasticClient.CreateIndex(RATINGS);
}
if (_elasticClient.IndexExists(SHARES).Exists == false)
{
_elasticClient.CreateIndex(SHARES);
}
if (_elasticClient.IndexExists(USER_LAYERS).Exists == false)
{
_elasticClient.CreateIndex(SHARES);
}
_logger.LogInformation("Finished initialing elasticsearch with uri: " + uri);
}
public void ConfigureIndexes(IElasticClient client) {
foreach (var index in GetIndexes()) {
IIndicesOperationResponse response = null;
int currentVersion = GetAliasVersion(client, index.Name);
var templatedIndex = index as ITemplatedElasticSeachIndex;
if (templatedIndex != null)
response = client.PutTemplate(index.VersionedName, template => templatedIndex.CreateTemplate(template).AddAlias(index.Name));
else if (!client.IndexExists(index.VersionedName).Exists)
response = client.CreateIndex(index.VersionedName, descriptor => index.CreateIndex(descriptor).AddAlias(index.Name));
Debug.Assert(response == null || response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the index or template.");
// Add existing indexes to the alias.
if (!client.AliasExists(index.Name).Exists) {
if (templatedIndex != null) {
var indices = client.IndicesStats().Indices.Where(kvp => kvp.Key.StartsWith(index.VersionedName)).Select(kvp => kvp.Key).ToList();
if (indices.Count > 0) {
var descriptor = new AliasDescriptor();
foreach (string name in indices)
descriptor.Add(add => add.Index(name).Alias(index.Name));
response = client.Alias(descriptor);
}
} else {
response = client.Alias(a => a.Add(add => add.Index(index.VersionedName).Alias(index.Name)));
}
Debug.Assert(response != null && response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the alias.");
}
// already on current version
if (currentVersion >= index.Version || currentVersion < 1)
continue;
// upgrade
_lockProvider.TryUsingAsync("reindex", async () => {
await _workItemQueue.EnqueueAsync(new ReindexWorkItem {
OldIndex = String.Concat(index.Name, "-v", currentVersion),
NewIndex = index.VersionedName,
Alias = index.Name,
DeleteOld = true
});
}, TimeSpan.Zero, CancellationToken.None);
}
}
public void Initialize(string uri = "http://localhost:9200/")
{
_logger.LogInformation("Initialing elastic search with uri: " + uri);
var pool = new SingleNodeConnectionPool(new Uri(uri));
var connectionString = new ConnectionSettings(
pool,
new HttpConnection(),
new ConnectionSettings.SourceSerializerFactory((builtin, settings) => new JsonNetSerializer(builtin, settings, () => new Newtonsoft.Json.JsonSerializerSettings
{
Converters = GeoJsonSerializer.Create(_geometryFactory, 3).Converters
})))
.PrettyJson();
_elasticClient = new ElasticClient(connectionString);
if (_elasticClient.IndexExists(OSM_POIS_INDEX1).Exists == false &&
_elasticClient.IndexExists(OSM_POIS_INDEX2).Exists == false)
{
CreatePointsOfInterestIndex(OSM_POIS_INDEX1);
_elasticClient.Alias(a => a.Add(add => add.Alias(OSM_POIS_ALIAS).Index(OSM_POIS_INDEX1)));
}
if (_elasticClient.IndexExists(OSM_POIS_INDEX1).Exists&&
_elasticClient.IndexExists(OSM_POIS_INDEX2).Exists)
{
_elasticClient.DeleteIndex(OSM_POIS_INDEX2);
}
if (_elasticClient.IndexExists(OSM_HIGHWAYS_INDEX1).Exists == false &&
_elasticClient.IndexExists(OSM_HIGHWAYS_INDEX2).Exists == false)
{
CreateHighwaysIndex(OSM_HIGHWAYS_INDEX1);
_elasticClient.Alias(a => a.Add(add => add.Alias(OSM_HIGHWAYS_ALIAS).Index(OSM_HIGHWAYS_INDEX1)));
}
if (_elasticClient.IndexExists(OSM_HIGHWAYS_INDEX1).Exists&&
_elasticClient.IndexExists(OSM_HIGHWAYS_INDEX2).Exists)
{
_elasticClient.DeleteIndex(OSM_HIGHWAYS_INDEX2);
}
if (_elasticClient.IndexExists(SHARES).Exists == false)
{
_elasticClient.CreateIndex(SHARES);
}
if (_elasticClient.IndexExists(CUSTOM_USER_LAYERS).Exists == false)
{
_elasticClient.CreateIndex(CUSTOM_USER_LAYERS);
}
_logger.LogInformation("Finished initialing elasticsearch with uri: " + uri);
}
/// <summary>
/// 重建索引
/// </summary>
/// <param name="client"></param>
/// <param name="oldIndexName">就索引名字</param>
/// <param name="newIndexName">新索引名字</param>
/// <param name="shardsNumber"></param>
/// <returns></returns>
public static bool ReIndex <T>(IElasticClient client, string oldIndexName, string newIndexName, int shardsNumber = 1) where T : class
{
//检查新索引别名
var alias = client.AliasExists($"{newIndexName}-reindex");
//如果改别名不存在
if (!alias.Exists)
{
//检查是否存在新索引
IExistsResponse newIndex = client.IndexExists(newIndexName);
client.UpdateIndexSettings(Indices.Index(newIndexName), setting => setting.IndexSettings(isetting => isetting.Setting("max_result_window", 100000)));
//如果新索引不存在则创建一个新索引，需要修改各项创建索引时才需要的参数则在这处理
if (!newIndex.Exists)
{
CreateIndexDescriptor createIndex = new CreateIndexDescriptor(newIndexName)
.Settings(s => s.NumberOfShards(shardsNumber).Analysis(a => a.Analyzers(aa => aa.Language("standard_listing", sa => sa.Language(Language.Chinese)))))
.Mappings(ms => ms.Map <T>(m => m.AutoMap()));
ICreateIndexResponse create = client.CreateIndex(createIndex);
Console.WriteLine($"{newIndexName}索引创建：{create.ApiCall.Success}");
//给新创建的索引添加一个索引别名表示时重新创建的
var r2 = client.Alias(t => t.Add(a => a.Index(newIndexName).Alias($"{newIndexName}-reindex")));
Console.WriteLine($"索引别名{newIndexName}=>{newIndexName}：{r2.ApiCall.Success}");
}
//将老索引中的文档索引导新建的索引
var r1 = client.ReindexOnServer(r => r.Source(s => s.Index(Indices.Index(oldIndexName))).Destination(d => d.Index(newIndexName)));
Console.WriteLine($"重新索引：{r1.ApiCall.Success}");
if (r1.ApiCall.Success)
{
//老索引中的所有文档全部到新索引之后删除老索引
client.DeleteIndex(Indices.Index(oldIndexName));
//新索引设置别名为老索引的名字
var r3 = client.Alias(t => t.Add(a => a.Index(newIndexName).Alias(oldIndexName)));
Console.WriteLine($"索引别名{newIndexName}=>{oldIndexName}：{r3.ApiCall.Success}");
}
}
return(alias.Exists);
}
public void SwitchAliasToIndex(string indexName)
{
var catAliases = _client.CatAliases(new CatAliasesRequest(_aliasName));
var oldIndexes = catAliases.Records.Select(x => x.Index);
var response = _client.Alias(a => RemoveIndexesFromAlias(a, oldIndexes).Add(i => i.Index(indexName).Alias(_aliasName)));
if (!response.IsValid)
{
throw new Exception("Failed while switching index alias.", response.OriginalException);
}
foreach (var index in oldIndexes)
{
_client.DeleteIndexAsync(new DeleteIndexRequest(index));
}
}
public bool MakeIndexCurrentAlias(string indexName)
{
//Get the Indices assigned to an Alias
string[] indicesToRemove = GetIndicesForAlias();
//So the alias command is a bulk operation and
//is not done on a single index or alias, but
//it is a series of add/remove commands of which
//require an alias name and an index. So we need
//to do a bit more with our command descriptor than
//we usually do. Always remember that for the compiler
//that
// command(x => x.Param());
// is the same as
// command(x => { x.Param(); return x });
var response = _client.Alias(a =>
{
//Add the new index
a.Add(add => add
.Index(indexName)
.Alias(this._config.AliasName)
);
//Remove any old ones
foreach (string oldIndex in indicesToRemove)
{
a.Remove(del => del
.Index(oldIndex)
.Alias(this._config.AliasName)
);
}
return(a);
});
if (!response.IsValid)
{
throw new Exception("Error swapping indices for alias: " + this._config.AliasName, response.OriginalException);
}
return(true);
}
public void ConfigureIndexes(IElasticClient client)
{
foreach (var index in GetIndexes())
{
IIndicesOperationResponse response = null;
int currentVersion = GetAliasVersion(client, index.Name);
var templatedIndex = index as ITemplatedElasticSeachIndex;
if (templatedIndex != null)
{
response = client.PutTemplate(index.VersionedName, template => templatedIndex.CreateTemplate(template).AddAlias(index.Name));
}
else if (!client.IndexExists(index.VersionedName).Exists)
{
response = client.CreateIndex(index.VersionedName, descriptor => index.CreateIndex(descriptor).AddAlias(index.Name));
}
Debug.Assert(response == null || response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the index or template.");
// Add existing indexes to the alias.
if (!client.AliasExists(index.Name).Exists)
{
if (templatedIndex != null)
{
var indices = client.IndicesStats().Indices.Where(kvp => kvp.Key.StartsWith(index.VersionedName)).Select(kvp => kvp.Key).ToList();
if (indices.Count > 0)
{
var descriptor = new AliasDescriptor();
foreach (string name in indices)
{
descriptor.Add(add => add.Index(name).Alias(index.Name));
}
response = client.Alias(descriptor);
}
}
else
{
response = client.Alias(a => a.Add(add => add.Index(index.VersionedName).Alias(index.Name)));
}
Debug.Assert(response != null && response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the alias.");
}
// already on current version
if (currentVersion >= index.Version || currentVersion < 1)
{
continue;
}
// upgrade
_lockProvider.TryUsingAsync("reindex", async() => {
await _workItemQueue.EnqueueAsync(new ReindexWorkItem {
OldIndex = String.Concat(index.Name, "-v", currentVersion),
NewIndex = index.VersionedName,
Alias = index.Name,
DeleteOld = true
});
}, TimeSpan.Zero, CancellationToken.None);
}
}
public virtual void ConfigureIndexes(IElasticClient client, IEnumerable <IElasticIndex> indexes = null)
{
if (indexes == null)
{
indexes = GetIndexes();
}
foreach (var idx in indexes)
{
int currentVersion = GetAliasVersion(client, idx.AliasName);
IIndicesOperationResponse response = null;
var templatedIndex = idx as ITemplatedElasticIndex;
if (templatedIndex != null)
{
response = client.PutTemplate(idx.VersionedName, template => templatedIndex.CreateTemplate(template).AddAlias(idx.AliasName));
}
else if (!client.IndexExists(idx.VersionedName).Exists)
{
response = client.CreateIndex(idx.VersionedName, descriptor => idx.CreateIndex(descriptor).AddAlias(idx.AliasName));
}
Debug.Assert(response == null || response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the index or template.");
// Add existing indexes to the alias.
if (!client.AliasExists(idx.AliasName).Exists)
{
if (templatedIndex != null)
{
var indices = client.IndicesStats().Indices.Where(kvp => kvp.Key.StartsWith(idx.VersionedName)).Select(kvp => kvp.Key).ToList();
if (indices.Count > 0)
{
var descriptor = new AliasDescriptor();
foreach (string name in indices)
{
descriptor.Add(add => add.Index(name).Alias(idx.AliasName));
}
response = client.Alias(descriptor);
}
}
else
{
response = client.Alias(a => a.Add(add => add.Index(idx.VersionedName).Alias(idx.AliasName)));
}
Debug.Assert(response != null && response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the alias.");
}
// already on current version
if (currentVersion >= idx.Version || currentVersion < 1)
{
continue;
}
var reindexWorkItem = new ReindexWorkItem {
OldIndex = String.Concat(idx.AliasName, "-v", currentVersion),
NewIndex = idx.VersionedName,
Alias = idx.AliasName,
DeleteOld = true,
ParentMaps = idx.GetIndexTypes()
.Select(kvp => new ParentMap {
Type = kvp.Value.Name, ParentPath = kvp.Value.ParentPath
})
.Where(m => !String.IsNullOrEmpty(m.ParentPath))
.ToList()
};
bool isReindexing = _lockProvider.IsLockedAsync(String.Concat("reindex:", reindexWorkItem.Alias, reindexWorkItem.OldIndex, reindexWorkItem.NewIndex)).Result;
// already reindexing
if (isReindexing)
{
continue;
}
// enqueue reindex to new version
_lockProvider.TryUsingAsync("enqueue-reindex", () => _workItemQueue.EnqueueAsync(reindexWorkItem), TimeSpan.Zero, CancellationToken.None).Wait();
}
}
public void OnSuccess(IElasticClient client, IIndicesOperationResponse response)
{
client.Alias(a => a.Add(aa => aa.Alias(client.Infer.DefaultIndex).Index(_indexName)));
}
