private static void CreateIndex(ElasticClient client, string filepath, string indexName)
{
Stopwatch timer = new Stopwatch();
timer.Start();
CreateIndexResponse createIndexResponse = client.Indices.Create(indexName, c => c
.Map <LogDocument>(m => m
.AutoMap <LogDocument>()
)
);
string[] lines = File.ReadAllLines(filepath);
int items = 0;
Parallel.ForEach(lines, (line) =>
{
if (!string.IsNullOrWhiteSpace(line))
{
client.CreateDocument <LogDocument>(new LogDocument()
{
Body = line.Trim()
});
items++;
}
});
timer.Stop();
Console.WriteLine($"Index created in {timer.ElapsedMilliseconds}ms with {items} element.");
}
private async Task EnsureIndexExists(string indexName, ElasticClient esClient)
{
ExistsResponse existsResult = await esClient.Indices.ExistsAsync(indexName).ConfigureAwait(false);
if (!existsResult.IsValid)
{
this.ReportEsRequestError(existsResult, "Index exists check");
}
if (existsResult.Exists)
{
return;
}
CreateIndexRequestBuilder createIndexRequestBuilder = new CreateIndexRequestBuilder(this.connectionData.Configuration, this.healthReporter);
CreateIndexResponse createIndexResult = await esClient.Indices.CreateAsync(indexName, createIndexRequestBuilder.BuildRequest).ConfigureAwait(false);
if (!createIndexResult.IsValid)
{
try
{
if (createIndexResult.ServerError?.Error?.Type != null &&
Regex.IsMatch(createIndexResult.ServerError.Error.Type, "index.*already.*exists.*exception", RegexOptions.IgnoreCase, TimeSpan.FromMilliseconds(500)))
{
// This is fine, someone just beat us to create a new index.
return;
}
}
catch (RegexMatchTimeoutException) { }
this.ReportEsRequestError(createIndexResult, "Create index");
}
}
public static bool CreateIndex <T>(IElasticClient elasticClient, string indexName) where T : class
{
var existsResponse = elasticClient.Indices.Exists(indexName);
// 存在则返回true 不存在创建
if (existsResponse.Exists)
{
return(true);
}
//基本配置
IIndexState indexState = new IndexState
{
Settings = new IndexSettings
{
NumberOfReplicas = 1, // 副本数
NumberOfShards = 6, // 分片数
},
};
CreateIndexResponse response = elasticClient.Indices.Create(indexName, p => p
.InitializeUsing(indexState).Map <T>(r => r.AutoMap())
);
return(response.IsValid);
}
/// <summary>
/// 如果同名索引不存在则创建索引
/// </summary>
/// <param name="client">ElasticClient实例</param>
/// <param name="indexName">要创建的索引名称</param>
/// <param name="numberOfReplicas">默认副本数量，如果是单实例，注意改成0</param>
/// <param name="numberOfShards">默认分片数量</param>
/// <returns></returns>
public static bool CreateIndex <T>(IElasticClient client, string indexName = "wizplant", int numberOfReplicas = 1, int numberOfShards = 5) where T : class
{
var existsResponse = client.Indices.Exists(indexName);
// 存在则返回true 不存在创建
if (existsResponse.Exists)
{
return(true);
}
var indexState = new IndexState
{
Settings = new IndexSettings
{
NumberOfReplicas = numberOfReplicas, //副本数
NumberOfShards = numberOfShards //分片数
}
};
if (string.IsNullOrWhiteSpace(indexName))
{
indexName = typeof(T).Name.ToLower();
}
CreateIndexResponse response = client.Indices.Create(indexName, p => p.InitializeUsing(indexState).Map <T>(r => r.AutoMap()));
// var result = client.CreateIndex(indexName, c => c.InitializeUsing(indexState).Mappings(ms => ms.Map<T>(m => m.AutoMap())));
return(response.IsValid);
}
public async Task <IActionResult> Post()
{
try
{
ExistsResponse indexExistsResponse = await _elasticClient.Indices.ExistsAsync(_appConfig.Elasticsearch.IndexName);
if (indexExistsResponse.IsValid && indexExistsResponse.Exists)
{
throw new Exception("Index of name " + _appConfig.Elasticsearch.IndexName + " already exists");
}
CreateIndexResponse response = await CreateIndex();
if (!response.IsValid)
{
_logger.LogError("Error creating index: " + response.DebugInformation);
return(StatusCode(StatusCodes.Status500InternalServerError));
}
return(Ok());
}
catch (Exception ex)
{
_logger.LogError(1, ex, "Unable to create Employees index; index already exists");
return(BadRequest());
}
}
/// <summary>
/// 创建索引
/// </summary>
/// <returns></returns>
public bool CreateElasticClientIndex(string dbname)
{
CreateIndexResponse createIndexResponse = client.Indices.Create(dbname);
var iscreate = createIndexResponse.Acknowledged;
return(iscreate);
}
/// <summary>
/// 创建索引
/// </summary>
/// <param name="indexName">索引名</param>
/// <param name="numberOfReplicas">副本数量</param>
/// <param name="numberOfShards">分片数量</param>
/// <returns></returns>
public async Task <CreateIndexResponse> CreateIndex(string indexName, int numberOfReplicas = 1, int numberOfShards = 5)
{
indexName = indexName.ToLower();
IIndexState indexState = new IndexState
{
Settings = new IndexSettings
{
NumberOfReplicas = numberOfReplicas,
NumberOfShards = numberOfShards
}
};
CreateIndexResponse response = await this.Client.Indices.CreateAsync(indexName, x => x.InitializeUsing(indexState).Map(m => m.AutoMap()));
return(response);
}
private static bool IsResposeValid(CreateIndexResponse response)
{
if (response.ServerError == null)
{
Console.WriteLine(response.Index);
Console.WriteLine("Index was created");
}
else
{
Console.WriteLine($"There is a error during creating index <{response.Index}>:");
Console.WriteLine(response.ServerError.Error.ToString());
return(false);
}
return(true);
}
/// <summary>
/// 创建索引
/// </summary>
/// <param name="elasticClient"></param>
public static CreateIndexResponse CreateIndex(this IElasticClient elasticClient, string indexName, int numberOfReplicas = 1, int numberOfShards = 5)
{
IIndexState indexState = new IndexState
{
Settings = new IndexSettings
{
NumberOfReplicas = numberOfReplicas,
// [副本数量]
NumberOfShards = numberOfShards
}
};
Func <CreateIndexDescriptor, ICreateIndexRequest> func = x => x.InitializeUsing(indexState).Map(m => m.AutoMap());
CreateIndexResponse response = elasticClient.Indices.Create(indexName, func);
return(response);
}
/// <summary>
/// 创建索引
/// </summary>
public CreateIndexResponse CreateIndex(string indexName, int numberOfReplicas = 1, int numberOfShards = 5)
{
IIndexState indexState = new IndexState
{
Settings = new IndexSettings
{
NumberOfReplicas = numberOfReplicas,
NumberOfShards = numberOfShards
}
};
ICreateIndexRequest Selector(CreateIndexDescriptor x) => x.InitializeUsing(indexState).Map <T>(ms => ms.AutoMap());
CreateIndexResponse response = ElasticClient.Indices.Create(indexName, Selector);
return(response);
}
/// <summary>
/// Unmarshaller the response from the service to the response class.
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
CreateIndexResponse response = new CreateIndexResponse();
context.Read();
int targetDepth = context.CurrentDepth;
while (context.ReadAtDepth(targetDepth))
{
if (context.TestExpression("Id", targetDepth))
{
var unmarshaller = StringUnmarshaller.Instance;
response.Id = unmarshaller.Unmarshall(context);
continue;
}
}
return(response);
}
public bool CreateIndex <T>(string indexName) where T : class
{
var existsResponse = B2BElasticClient.Indices.Exists(indexName);
if (existsResponse.Exists)
{
return(true);
}
IIndexState indexState = new IndexState
{
Settings = new IndexSettings
{
NumberOfReplicas = 1, //副本数
NumberOfShards = 2, //分片数
}
};
CreateIndexResponse response = B2BElasticClient.Indices.Create(indexName, p => p.InitializeUsing(indexState).Map <T>(m => m.AutoMap()));
return(response.IsValid);
}
public static void CreateIndex(IElasticClient client)
{
//client.Indices.DeleteAsync("release").Result;
//client.Indices.GetAsync(new GetIndexRequest(Indices.All)).Result;
CreateIndexResponse createIndexResponse = client.Indices.Create("release", c => c
.Settings(s => s
.Analysis(a => a
.Analyzers(aa => aa
.Custom("rlsname_analyzer", ca => ca
.Tokenizer("rlsname_tokenizer")
.Filters("lowercase")
)
)
.Tokenizers(at => at
.Pattern("rlsname_tokenizer", pt => pt
.Pattern("[^A-Za-z0-9]+"))
)
.Normalizers(an => an
.Custom("lowercase_normalizer", cn => cn
.CharFilters()
.Filters("lowercase"))
)
)
)
.Map <ElasticRelease>(mm => mm
.Properties(p => p
.Text(t => t
.Name(n => n.Title)
.Analyzer("rlsname_analyzer")
.Fields(tf => tf
.Keyword(kw => kw
.Name("keyword")
.Normalizer("lowercase_normalizer")
)
)
)
)
)
);
}
/// <summary>
/// Create a new index in ES
/// </summary>
public bool CreateIndex <T1, T2>(string indName) where T1 : class where T2 : class
{
CreateIndexResponse resp = null;
try
{
if (!_client.Indices.Exists(indName).Exists)
{
resp = _client.Indices.Create(indName, c => c.Map <T1>(m => m.AutoMap()).Map <T2>(m => m.AutoMap()));
}
_logger.LogInformation($"Index {indName} has been created");
}
catch (Exception)
{
_logger.LogInformation($"An error occurred while creating the index {indName}");
return(false);
}
return(resp.IsValid);
}
public async Task <IActionResult> AnonymousPost()
{
try
{
_logger.LogInformation("Uri: " + _appConfig.Elasticsearch.Uri);
_logger.LogInformation("IndexName: " + _appConfig.Elasticsearch.IndexName);
_logger.LogInformation("Username: "******"Password: "******"Index DOES NOT exist. Creating index...");
CreateIndexResponse response = await CreateIndex();
if (!response.IsValid)
{
_logger.LogError("Error creating index: " + response.DebugInformation);
return(StatusCode(StatusCodes.Status500InternalServerError));
}
}
else
{
if (indexExistsResponse.Exists)
{
_logger.LogInformation("Index already exists. Skipping index creation...");
}
}
return(Ok());
}
catch (Exception ex)
{
_logger.LogError(1, ex, "Unable to create Employees index");
return(StatusCode(StatusCodes.Status500InternalServerError));
}
}
public async Task <bool> CreateIndex(List <T> entity)
{
var indexIsExists = await _elasticClient.Indices.ExistsAsync(IndexName);
if (indexIsExists.Exists)
{
await _elasticClient.Indices.DeleteAsync(IndexName);
}
CreateIndexResponse createIndex = await _elasticClient.Indices.CreateAsync(IndexName, p => p
.Map(r => r.AutoMap()));
var indexIsValid = createIndex.IsValid;
if (indexIsValid)
{
var indexCreated = await _elasticClient.IndexManyAsync(entity, index : IndexName);
return(indexCreated.IsValid);
}
return(false);
}
static void Main(string[] args)
{
_indexName = "TEST";
_indexName = _indexName.ToLower();//索引名称一定要小写
_elasticClient = GetElasticClientByPool();
var existsResponse = _elasticClient.Indices.Exists(_indexName);
if (!existsResponse.Exists)
{
//基本配置
IIndexState indexState = new IndexState
{
Settings = new IndexSettings
{
NumberOfReplicas = 1, //副本数
NumberOfShards = 6 //分片数
}
};
CreateIndexResponse response = _elasticClient.Indices.Create(_indexName, p => p
.InitializeUsing(indexState)
.Map <People>(r => r.AutoMap())
);
if (response.IsValid)
{
Console.WriteLine("索引创建成功");
}
else
{
Console.WriteLine("索引创建失败");
}
}
Console.ReadLine();
}
private async Task InitClientAsync()
{
// Setup Serilog to log to the console
Log.Logger =
new LoggerConfiguration()
.WriteTo.Console()
.CreateLogger();
// You load this in your startup.cs (either in the constructor or before you configure your
// services in ConfigureServices
_appConfig =
new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build()
.Get <AppConfig>();
// Once loaded inject the ElasticsearchConfig into the IoC container
//services.AddSingleton(appConfig.Elasticsearch)
// Now that you've added the ElasticsearchConfig to your IoC container it will be populated
// via the constructor (e.g. your repository constructor)
ConnectionSettings settings =
new ConnectionSettings(_appConfig.Elasticsearch.Uri)
.DefaultMappingFor <EpFile>(m => m.IndexName(_appConfig.Elasticsearch.EpFileIndex));
_esClient = new ElasticClient(settings);
await _esClient.Indices.DeleteAsync(_appConfig.Elasticsearch.BookIndex);
CreateIndexResponse response = await _esClient.Indices.CreateAsync(
_appConfig.Elasticsearch.BookIndex,
c => c.Map <Book>(m => m.AutoMap()));
Console.WriteLine(response.Index);
}
public static void CreateIndex(ElasticClient client, IndexType idxTyp)
{
IndexSettings set = new IndexSettings();
set.NumberOfReplicas = 2;
if (idxTyp == IndexType.DataSource)
{
set.NumberOfShards = 4;
}
else
{
set.NumberOfShards = 8;
}
// Add the Analyzer with a name
set.Analysis = new Nest.Analysis()
{
Analyzers = new Analyzers(),
TokenFilters = BasicTokenFilters(),
};
set.Analysis.Analyzers.Add("default", DefaultAnalyzer());
IndexState idxSt = new IndexState();
idxSt.Settings = set;
CreateIndexResponse res = null;
switch (idxTyp)
{
case IndexType.VerejneZakazky:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i
.InitializeUsing(idxSt)
.Map <Lib.Data.VZ.VerejnaZakazka>(map => map.AutoMap().DateDetection(false))
);
break;
case IndexType.ProfilZadavatele:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i
.InitializeUsing(idxSt)
.Map <Lib.Data.VZ.ProfilZadavatele>(map => map.AutoMap().DateDetection(false))
);
break;
case IndexType.Insolvence:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i
.InitializeUsing(idxSt)
.Map <Lib.Data.Insolvence.Rizeni>(map => map.AutoMap().DateDetection(false))
);
break;
case IndexType.Dotace:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i
.InitializeUsing(idxSt)
.Map <Data.Dotace.Dotace>(map => map.AutoMap().DateDetection(false))
);
break;
case IndexType.Osoby:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i
.InitializeUsing(new IndexState()
{
Settings = new IndexSettings()
{
NumberOfReplicas = 2,
NumberOfShards = 2,
Analysis = new Nest.Analysis()
{
TokenFilters = BasicTokenFilters(),
Analyzers = new Analyzers(new Dictionary <string, IAnalyzer>()
{
["default"] = DefaultAnalyzer(),
["lowercase"] = LowerCaseOnlyAnalyzer(),
["lowercase_ascii"] = LowerCaseAsciiAnalyzer()
})
}
}
})
.Map <Data.OsobyES.OsobaES>(map => map
.AutoMap()
.Properties(p => p
.Text(t => t
.Name(n => n.FullName)
.Fields(ff => ff
.Text(tt => tt
.Name("lower")
.Analyzer("lowercase")
)
.Text(tt => tt
.Name("lowerascii")
.Analyzer("lowercase_ascii")
)
)
)
)
)
);
break;
case IndexType.Smlouvy:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i
.InitializeUsing(idxSt)
.Map <Lib.Data.Smlouva>(map => map.AutoMap().DateDetection(false))
);
break;
case IndexType.Firmy:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i
.InitializeUsing(idxSt)
.Map <Data.Firma.Search.FirmaInElastic>(map => map.AutoMap(maxRecursion: 1))
);
break;
case IndexType.KIndex:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i
.InitializeUsing(idxSt)
.Map <Analysis.KorupcniRiziko.KIndexData>(map => map.AutoMap(maxRecursion: 2))
);
break;
case IndexType.KIndexTemp:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i
.InitializeUsing(idxSt)
.Map <Analysis.KorupcniRiziko.KIndexData>(map => map.AutoMap(maxRecursion: 2))
);
break;
case IndexType.KIndexBackup:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i
.InitializeUsing(idxSt)
.Map <Analysis.KorupcniRiziko.Backup>(map => map.AutoMap(maxRecursion: 2))
);
break;
case IndexType.KIndexBackupTemp:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i
.InitializeUsing(idxSt)
.Map <Analysis.KorupcniRiziko.Backup>(map => map.AutoMap(maxRecursion: 2))
);
break;
case IndexType.KindexFeedback:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i
.InitializeUsing(idxSt)
.Map <Analysis.KorupcniRiziko.KindexFeedback>(map => map.AutoMap(maxRecursion: 2))
);
break;
case IndexType.Logs:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i
.InitializeUsing(idxSt)
.Map <Lib.Data.Logs.ProfilZadavateleDownload>(map => map.AutoMap(maxRecursion: 1))
);
break;
case IndexType.Audit:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i
.InitializeUsing(new IndexState()
{
Settings = new IndexSettings()
{
NumberOfReplicas = 1,
NumberOfShards = 2
}
}
)
.Map <Lib.Data.Audit>(map => map.AutoMap(maxRecursion: 1))
);
break;
case IndexType.VerejneZakazkyNaProfiluRaw:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i
.InitializeUsing(idxSt)
.Map <Lib.Data.External.ProfilZadavatelu.ZakazkaRaw>(map => map
.Properties(p => p
.Keyword(k => k.Name(n => n.ZakazkaId))
.Keyword(k => k.Name(n => n.Profil))
.Date(k => k.Name(n => n.LastUpdate))
)
)
);
break;
case IndexType.RPP_Kategorie:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i
.InitializeUsing(new IndexState()
{
Settings = new IndexSettings()
{
NumberOfReplicas = 2,
NumberOfShards = 2
}
}
)
.Map <Lib.Data.External.RPP.KategorieOVM>(map => map.AutoMap(maxRecursion: 1))
);
break;
case IndexType.RPP_OVM:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i
.InitializeUsing(new IndexState()
{
Settings = new IndexSettings()
{
NumberOfReplicas = 2,
NumberOfShards = 2
}
}
)
.Map <Lib.Data.External.RPP.OVMFull>(map => map.AutoMap(maxRecursion: 1))
);
break;
case IndexType.RPP_ISVS:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i
.InitializeUsing(new IndexState()
{
Settings = new IndexSettings()
{
NumberOfReplicas = 2,
NumberOfShards = 2
}
}
)
.Map <Lib.Data.External.RPP.ISVS>(map => map.AutoMap(maxRecursion: 1))
);
break;
}
}
public static void CreateIndex(ElasticClient client, IndexType idxTyp)
{
IndexSettings set = new IndexSettings();
set.NumberOfReplicas = 2;
if (idxTyp == IndexType.DataSource)
{
set.NumberOfShards = 4;
}
else
{
set.NumberOfShards = 8;
}
// Create a Custom Analyzer ...
var an = new CustomAnalyzer();
an.Tokenizer = "standard";
// ... with Filters from the StandardAnalyzer
var filter = new List <string>();
filter.Add("lowercase");
filter.Add("czech_stop");
//an.Filter.Add("czech_keywords");
filter.Add("czech_stemmer"); //pouzit Hunspell
filter.Add("asciifolding");
an.Filter = filter;
// Add the Analyzer with a name
set.Analysis = new Nest.Analysis()
{
Analyzers = new Analyzers(),
TokenFilters = new TokenFilters(),
};
set.Analysis.Analyzers.Add("default", an);
set.Analysis.TokenFilters.Add("czech_stop", new StopTokenFilter()
{
StopWords = new string[] { "_czech_" }
});
set.Analysis.TokenFilters.Add("czech_stemmer", new StemmerTokenFilter()
{
Language = "czech"
}); //Humspell
IndexState idxSt = new IndexState();
idxSt.Settings = set;
CreateIndexResponse res = null;
switch (idxTyp)
{
case IndexType.VerejneZakazky:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i
.InitializeUsing(idxSt)
.Map <Lib.Data.VZ.VerejnaZakazka>(map => map.AutoMap().DateDetection(false))
);
break;
case IndexType.ProfilZadavatele:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i
.InitializeUsing(idxSt)
.Map <Lib.Data.VZ.ProfilZadavatele>(map => map.AutoMap().DateDetection(false))
);
break;
case IndexType.Insolvence:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i //todo: es7 check
.InitializeUsing(idxSt)
.Map <Lib.Data.Insolvence.Rizeni>(map => map.AutoMap().DateDetection(false))
);
break;
case IndexType.Dotace:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i //todo: es7 check
.InitializeUsing(idxSt)
.Map <Data.Dotace.Dotace>(map => map.AutoMap().DateDetection(false))
);
break;
case IndexType.Smlouvy:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i //todo: es7 check
.InitializeUsing(idxSt)
.Map <Lib.Data.Smlouva>(map => map.AutoMap().DateDetection(false))
);
break;
case IndexType.Firmy:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i //todo: es7 check
.InitializeUsing(idxSt)
.Map <Data.Firma.Search.FirmaInElastic>(map => map.AutoMap(maxRecursion: 1))
);
break;
case IndexType.Logs:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i //todo: es7 check
.InitializeUsing(idxSt)
.Map <Lib.Data.Logs.ProfilZadavateleDownload>(map => map.AutoMap(maxRecursion: 1))
);
break;
case IndexType.VerejneZakazkyNaProfiluRaw:
res = client.Indices
.Create(client.ConnectionSettings.DefaultIndex, i => i //todo: es7 check
.InitializeUsing(idxSt)
.Map <Lib.Data.External.ProfilZadavatelu.ZakazkaRaw>(map => map
.Properties(p => p
.Keyword(k => k.Name(n => n.ZakazkaId))
.Keyword(k => k.Name(n => n.Profil))
.Date(k => k.Name(n => n.LastUpdate))
)
)
);
break;
}
}
