public static bool CreateIndex <T>(IElasticClient elasticClient, string indexName) where T : class
{
var existsResponse = elasticClient.IndexExists(indexName);
// 存在则返回true 不存在创建
if (existsResponse.Exists)
{
return(true);
}
//基本配置
IIndexState indexState = new IndexState
{
Settings = new IndexSettings
{
NumberOfReplicas = 1, //副本数
NumberOfShards = 6 //分片数
}
};
ICreateIndexResponse response = elasticClient.CreateIndex(indexName, p => p
.InitializeUsing(indexState).Mappings(m => m.Map <T>(r => r.AutoMap()))
);
return(response.IsValid);
}
public void CreateIndex(IndexDefinition index, bool autoReindex = true)
{
DeleteIndexIfExists(index);
ICreateIndexResponse createIndexResponse = index.Create(_client);
if (autoReindex)
{
Reindex(index);
}
}
public void Log(LogRecord record)
{
// Convert to es log record
var logRecord = new ESLogRecord
{
LogID = record.LogID,
Namespace = record.Namespace,
UniqueKey = record.UniqueKey,
AppKey = record.AppKey,
Environment = record.Environment,
Host = record.Host,
HostIP = record.HostIP,
Token = record.UserToken,
UserId = record.UserId,
UserIP = record.UserIP,
Username = record.Username,
IdentityKey = record.IdentityKey,
Level = record.Level,
Process = record.Process,
ThreadId = record.ThreadId,
Message = record.Message,
Exception = record.ExceptionString,
Data = record.DataString,
TimeStamp = record.TimeStamp
};
// Get the index name
string indexName = GetIndexName(typeof(ESLogRecord));
try
{
// Create index if necessary
ICreateIndexResponse createIndexResponse = null;
if (!_client.IndexExists(indexName).Exists)
{
// Create the index descriptor
var descriptor = GetCreateIndexDescriptor(indexName, typeof(ESLogRecord));
createIndexResponse = _client.CreateIndex(descriptor);
}
if (createIndexResponse == null || !createIndexResponse.IsValid)
{
// Error logging
}
// Index the incoming data
var indexDataResponse = _client.Index <ESLogRecord>(logRecord, idx => idx.Index(indexName));
}
catch (Exception ex)
{
}
}
public async Task <IndexResponseDTO> CreateIndexAsync(string indexName, string aliasName)
{
CreateIndexDescriptor createIndexDescriptor = new CreateIndexDescriptor(indexName)
.Mappings(ms => ms
.Map <T>(m => m.AutoMap())
)
.Aliases(a => a.Alias(aliasName));
ICreateIndexResponse response = await elasticClient.CreateIndexAsync(createIndexDescriptor);
return(new IndexResponseDTO()
{
IsValid = response.IsValid,
StatusMessage = response.DebugInformation,
Exception = response.OriginalException
});
}
private async Task EnsureIndexExists(string indexName, ElasticClient esClient)
{
IExistsResponse existsResult = await esClient.IndexExistsAsync(indexName).ConfigureAwait(false);
if (!existsResult.IsValid)
{
this.ReportEsRequestError(existsResult, "Index exists check");
}
if (existsResult.Exists)
{
return;
}
// TODO: allow the consumer to fine-tune index settings
IndexState indexSettings = new IndexState();
indexSettings.Settings = new IndexSettings();
indexSettings.Settings.NumberOfReplicas = this.connectionData.Configuration.NumberOfReplicas;
indexSettings.Settings.NumberOfShards = this.connectionData.Configuration.NumberOfShards;
indexSettings.Settings.Add("refresh_interval", this.connectionData.Configuration.RefreshInterval);
if (this.connectionData.Configuration.DefaultPipeline != null)
{
indexSettings.Settings.Add("default_pipeline", this.connectionData.Configuration.DefaultPipeline);
}
ICreateIndexResponse createIndexResult = await esClient.CreateIndexAsync(indexName, c => c.InitializeUsing(indexSettings)).ConfigureAwait(false);
if (!createIndexResult.IsValid)
{
try
{
if (createIndexResult.ServerError?.Error?.Type != null &&
Regex.IsMatch(createIndexResult.ServerError.Error.Type, "index.*already.*exists.*exception", RegexOptions.IgnoreCase, TimeSpan.FromMilliseconds(500)))
{
// This is fine, someone just beat us to create a new index.
return;
}
}
catch (RegexMatchTimeoutException) { }
this.ReportEsRequestError(createIndexResult, "Create index");
}
}
public async override Task <int> Write(LogMessage message)
{
var affrows = 0;
try
{
IndexName indexName = message.ProjectName.ToLower();
TypeName typeName = message.LogLevel.ToString().ToLower();
if (!client.IndexExists(indexName).Exists)
{
IIndexState indexState = new IndexState
{
Settings = new IndexSettings
{
NumberOfReplicas = 1, //副本数
NumberOfShards = 6 //分片数
}
};
ICreateIndexResponse response = client.CreateIndex(indexName, p => p
.InitializeUsing(indexState)
.Mappings(m => m.Map <LogMessage>(r => r.AutoMap()))
);
}
IndexRequest <LogMessage> req = new IndexRequest <LogMessage>(indexName, typeName);
var res = await client.IndexAsync(message, selector => selector.Type(typeName).Index(indexName));
if (res.Result == Result.Created)
{
affrows = 1;
}
}
catch (Exception ex)
{
logger.LogError(ex.Message, ex);
}
return(affrows);
}
/// <summary>
/// Creates the index asynchronous.
/// </summary>
public async Task <string> CreateIndexAsync <TIndexType>()
where TIndexType : class
{
string indexName = Guid.NewGuid().ToString();
ICreateIndexResponse createIndexResponse = await Client.CreateIndexAsync(
index : indexName,
selector : descriptor => descriptor
.Mappings(mappingsDescriptor => mappingsDescriptor
.Map <TIndexType>(mappingDescriptor => mappingDescriptor
.AutoMap <TIndexType>())));
if (!createIndexResponse.IsValid)
{
throw new InvalidOperationException(
$"Could not create the index: {createIndexResponse.DebugInformation}",
createIndexResponse.OriginalException);
}
return(indexName);
}
public async Task <bool> CreateIndexAsync(string indexName)
{
var createIndexDescriptor = new CreateIndexDescriptor(indexName)
.Mappings(ms => ms
.Map <Product>(m => m
.AutoMap()
.Properties(ps => ps
.Completion(c => c
.Name(p => p.Suggest))))
);
if (_elasticClient.IndexExists(indexName.ToLowerInvariant()).Exists)
{
_elasticClient.DeleteIndex(indexName.ToLowerInvariant());
}
ICreateIndexResponse createIndexResponse = await _elasticClient.CreateIndexAsync(createIndexDescriptor);
return(createIndexResponse.IsValid);
}
public void CreateIndex(string indexName)
{
if (!_elasticClient.IndexExists(indexName).Exists)
{
var indexSettings = new IndexSettings();
indexSettings.NumberOfReplicas = 1;
indexSettings.NumberOfShards = 3;
var createIndexDescriptor = new CreateIndexDescriptor(indexName)
.Mappings(ms => ms
.Map <LogDto>(m => m
.AutoMap()))
.InitializeUsing(new IndexState()
{
Settings = indexSettings
})
.Aliases(x => x.Alias(indexName));
ICreateIndexResponse createIndexResponse = _elasticClient.CreateIndex(createIndexDescriptor);
}
}
public void CreateIndex()
{
IndexSettings indexSettings = new IndexSettings();
indexSettings.NumberOfReplicas = 1;
indexSettings.NumberOfShards = 1;
IndexState indexState = new IndexState
{
Settings = indexSettings
};
if (client.IndexExists(INDEXNAME_VOTERSEARCHER).Exists)
{
Console.WriteLine("Index already exists...deleting existing index...");
client.DeleteIndex(INDEXNAME_VOTERSEARCHER);
//return;
}
ICreateIndexResponse response = client.CreateIndex(INDEXNAME_VOTERSEARCHER, c => c
.InitializeUsing(indexState)
.Mappings(m => m.Map <VoterInfo>(mp => mp.AutoMap())));
}
/// <summary>
/// Creates configured ElasticSearch client.
/// </summary>
/// <returns>
/// Returnes configured ElasticSearch client.
/// </returns>
public IElasticClient CreateElasticClient()
{
string indexName = ConfigurationManager.AppSettings["elasticSearchIndexName"];
string elasticSearchUri = ConfigurationManager.AppSettings["elasticSearchUri"];
ElasticClient client = new ElasticClient(
new ConnectionSettings(new Uri(elasticSearchUri)));
if (client.IndexExists(indexName).Exists)
{
return client;
}
ICreateIndexResponse createIndexResponse = client.CreateIndex(indexName, u => u
.Settings(s => s
.Analysis(a => a
.Tokenizers(token => token
.NGram("customNGramTokenizer", ng => ng
.MinGram(1)
.MaxGram(15)
.TokenChars(TokenChar.Letter, TokenChar.Digit)))
.Analyzers(analyzer => analyzer
.Custom("customIndexNgramAnalyzer", cia => cia
.Filters("lowercase")
.Tokenizer("customNGramTokenizer"))
.Custom("customSearchNgramAnalyzer", csa => csa
.Filters("lowercase")
.Tokenizer("keyword")))))
.Mappings(map => map
.Map<DocumentEntity>(m => m
.AutoMap()
)
)
);
return client;
}
private async Task EnsureIndexExists(string currentIndexName, ElasticClient esClient)
{
IExistsResponse existsResult = await esClient.IndexExistsAsync(currentIndexName);
if (!existsResult.IsValid)
{
this.ReportEsRequestError(existsResult, "Index exists check");
}
if (existsResult.Exists)
{
return;
}
// TODO: allow the consumer to fine-tune index settings
IndexState indexState = new IndexState();
indexState.Settings = new IndexSettings();
indexState.Settings.NumberOfReplicas = 1;
indexState.Settings.NumberOfShards = 5;
indexState.Settings.Add("refresh_interval", "15s");
ICreateIndexResponse createIndexResult = await esClient.CreateIndexAsync(currentIndexName, c => c.InitializeUsing(indexState));
if (!createIndexResult.IsValid)
{
if (createIndexResult.ServerError != null &&
createIndexResult.ServerError.Error != null &&
string.Equals(createIndexResult.ServerError.Error.Type, "IndexAlreadyExistsException", StringComparison.OrdinalIgnoreCase))
{
// This is fine, someone just beat us to create a new index.
return;
}
this.ReportEsRequestError(createIndexResult, "Create index");
}
}
static void Main(string[] args)
{
_indexName = "TEST";
_indexName = _indexName.ToLower();//索引名称一定要小写
_elasticClient = GetElasticClientByPool();
var existsResponse = _elasticClient.IndexExists(_indexName);
if (!existsResponse.Exists)
{
//基本配置
IIndexState indexState = new IndexState
{
Settings = new IndexSettings
{
NumberOfReplicas = 1, //副本数
NumberOfShards = 6 //分片数
}
};
ICreateIndexResponse response = _elasticClient.CreateIndex(_indexName, p => p
.InitializeUsing(indexState)
.Mappings(m => m.Map <People>(r => r.AutoMap()))
);
if (response.IsValid)
{
Console.WriteLine("索引创建成功");
}
else
{
Console.WriteLine("索引创建失败");
}
}
Console.ReadLine();
}
/// <summary>
/// 重建索引
/// </summary>
/// <param name="client"></param>
/// <param name="oldIndexName">就索引名字</param>
/// <param name="newIndexName">新索引名字</param>
/// <param name="shardsNumber"></param>
/// <returns></returns>
public static bool ReIndex <T>(IElasticClient client, string oldIndexName, string newIndexName, int shardsNumber = 1) where T : class
{
//检查新索引别名
var alias = client.AliasExists($"{newIndexName}-reindex");
//如果改别名不存在
if (!alias.Exists)
{
//检查是否存在新索引
IExistsResponse newIndex = client.IndexExists(newIndexName);
client.UpdateIndexSettings(Indices.Index(newIndexName), setting => setting.IndexSettings(isetting => isetting.Setting("max_result_window", 100000)));
//如果新索引不存在则创建一个新索引，需要修改各项创建索引时才需要的参数则在这处理
if (!newIndex.Exists)
{
CreateIndexDescriptor createIndex = new CreateIndexDescriptor(newIndexName)
.Settings(s => s.NumberOfShards(shardsNumber).Analysis(a => a.Analyzers(aa => aa.Language("standard_listing", sa => sa.Language(Language.Chinese)))))
.Mappings(ms => ms.Map <T>(m => m.AutoMap()));
ICreateIndexResponse create = client.CreateIndex(createIndex);
Console.WriteLine($"{newIndexName}索引创建：{create.ApiCall.Success}");
//给新创建的索引添加一个索引别名表示时重新创建的
var r2 = client.Alias(t => t.Add(a => a.Index(newIndexName).Alias($"{newIndexName}-reindex")));
Console.WriteLine($"索引别名{newIndexName}=>{newIndexName}：{r2.ApiCall.Success}");
}
//将老索引中的文档索引导新建的索引
var r1 = client.ReindexOnServer(r => r.Source(s => s.Index(Indices.Index(oldIndexName))).Destination(d => d.Index(newIndexName)));
Console.WriteLine($"重新索引：{r1.ApiCall.Success}");
if (r1.ApiCall.Success)
{
//老索引中的所有文档全部到新索引之后删除老索引
client.DeleteIndex(Indices.Index(oldIndexName));
//新索引设置别名为老索引的名字
var r3 = client.Alias(t => t.Add(a => a.Index(newIndexName).Alias(oldIndexName)));
Console.WriteLine($"索引别名{newIndexName}=>{oldIndexName}：{r3.ApiCall.Success}");
}
}
return(alias.Exists);
}
/// <summary>
///
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="client"></param>
/// <param name="indexName">索引名</param>
/// <param name="newIndexName">新索引名，重建索引用</param>
/// <param name="max_result_window"></param>
/// <param name="shardsNumber">分片数</param>
/// <returns></returns>
public static IElasticClient CreateOrUpdateIndex <T>(this IElasticClient client, string indexName, string newIndexName = null, long max_result_window = 100000, int shardsNumber = 1) where T : class
{
try
{
if (!string.IsNullOrWhiteSpace(newIndexName))
{
var x = ReIndex <T>(client, indexName, newIndexName, shardsNumber);
if (x == false)
{
indexName = newIndexName;
}
}
IExistsResponse index = client.IndexExists(indexName);
client.UpdateIndexSettings(Indices.Index(indexName), setting => setting.IndexSettings(isetting => isetting.Setting("max_result_window", max_result_window)));
if (!index.Exists)
{
CreateIndexDescriptor newIndex = new CreateIndexDescriptor(indexName)
.Settings(s => s.NumberOfShards(shardsNumber).NumberOfReplicas(1).Analysis(a => a.Analyzers(aa => aa.Language("standard_listing", sa => sa.Language(Language.Chinese)))))
.Mappings(ms => ms.Map <T>(m => m.AutoMap()));
ICreateIndexResponse create = client.CreateIndex(newIndex);
Console.WriteLine($"[{indexName}]\t索引已创建。");
}
else
{
IPutMappingResponse putMap = client.Map <T>(m => m.AutoMap());
if (putMap.ApiCall.Success)
{
Console.WriteLine($"[{indexName}]\t索引已更新。");
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
return(client);
}
protected void EnsureModelIndices(ElasticClient client)
{
base.ExecuteMethod("EnsureModelIndices", delegate()
{
if (debug_reset)
{
bool executeDebug = false;
lock (debug_lock)
{
if (debug_reset)
{
executeDebug = true;
}
debug_reset = false;
}
if (executeDebug)
{
//client.Map<Objective>(m => m
// .MapFromAttributes()
// .Type(DocumentTypes.OBJECTIVES)
// .Properties(props => props
// .String(s => s
// .Name(p => p.campaign_id)
// .Index(FieldIndexOption.NotAnalyzed))
// .String(s => s
// .Name(p => p.objective_id)
// .Index(FieldIndexOption.NotAnalyzed)
// ))
// );
}
}
if (!this.HasEnsuredModelIndices)
{
lock (ensure_lock)
{
if (!this.HasEnsuredModelIndices)
{
if (!client.IndexExists(this.IndexName).Exists)
{
CustomAnalyzer ignoreCaseAnalyzer = new CustomAnalyzer
{
Tokenizer = "keyword",
Filter = new[] { "lowercase" }
};
Analysis analysis = new Analysis();
analysis.Analyzers = new Analyzers();
analysis.Analyzers.Add("case_insensitive", ignoreCaseAnalyzer);
ICreateIndexResponse createResult = client.CreateIndex(this.IndexName, delegate(Nest.CreateIndexDescriptor descriptor)
{
descriptor.Settings(ss => ss
.Analysis(a => analysis)
.NumberOfReplicas(this.ReplicaCount)
.NumberOfShards(this.ShardCount)
.Setting("merge.policy.merge_factor", "10")
.Setting("search.slowlog.threshold.fetch.warn", "1s")
.Setting("max_result_window", "2147483647")
);
this.MapIndexModels(descriptor);
return(descriptor);
});
if (!createResult.Acknowledged)
{
throw new Exception("Error creating index, mapping is no longer valid");
}
}
HasEnsuredModelIndices = true;
}
}
}
});
}
