[HttpGet("import")] // todo this endpoint is test only...
public async Task <IActionResult> Import()
{
var products = await _productsService.GetProducts();
var productsToUpdate = products.Where(x => x.IsActive && !x.IsDeleted)
.Select(x => new ProductSearchModel(x)).ToList();
var productsToDelete = products.Where(x => !x.IsActive || x.IsDeleted)
.Select(x => new ProductSearchModel(x)).ToList();
if (productsToUpdate.Any())
{
var updateResponse = await _client.IndexManyAsync(productsToUpdate, IndexName.From <ProductSearchModel>());
if (updateResponse.OriginalException != null)
{
throw updateResponse.OriginalException;
}
}
if (productsToDelete.Any())
{
var deleteResponse = await _client.DeleteManyAsync(productsToDelete, IndexName.From <ProductSearchModel>());
if (deleteResponse.OriginalException != null)
{
throw deleteResponse.OriginalException;
}
}
return(Accepted());
}
public async Task Consume(ConsumeContext <Batch <ProductUpdatedEvent> > context)
{
var productsWithoutDuplicates = context.Message.GroupBy(x => x.Message.ProductDetails.Id)
.Select(x => x.OrderByDescending(y => y.Message.UpdateDateUtc).FirstOrDefault())
.ToList();
var productsToUpdate = productsWithoutDuplicates.Where(x => x.Message.ProductDetails.IsActive && !x.Message.ProductDetails.IsDeleted)
.Select(x => new ProductSearchModel(x.Message.ProductDetails)).ToList();
var productsToDelete = productsWithoutDuplicates.Where(x => !x.Message.ProductDetails.IsActive || x.Message.ProductDetails.IsDeleted)
.Select(x => new ProductSearchModel(x.Message.ProductDetails)).ToList();
if (productsToUpdate.Any())
{
var updateResponse = await _client.IndexManyAsync(productsToUpdate, IndexName.From <ProductSearchModel>());
if (updateResponse.OriginalException != null)
{
throw updateResponse.OriginalException;
}
}
if (productsToDelete.Any())
{
var deleteResponse = await _client.DeleteManyAsync(productsToDelete, IndexName.From <ProductSearchModel>());
if (deleteResponse.OriginalException != null)
{
throw deleteResponse.OriginalException;
}
}
}
/** === An example with requests */
[U] public void UsingWithRequests()
{
/* Given the following CLR type that describes a document */
var project = new Project {
Name = "hello-world"
};
/** we can see an example of how `DocumentPath` helps your describe your requests more tersely */
var request = new IndexRequest <Project>(2)
{
Document = project
};
request = new IndexRequest <Project>(project)
{
};
/** when comparing with the full blown constructor and passing document manually,
* `DocumentPath<T>`'s benefits become apparent.
*/
request = new IndexRequest <Project>(IndexName.From <Project>(), TypeName.From <Project>(), 2)
{
Document = project
};
}
public SampleElasticsearchClient(string endPoint, Amazon.RegionEndpoint regionEndPoint)
{
AWSCredentials awsCredentials = FallbackCredentialsFactory.GetCredentials();
AwsHttpConnection conn = new AwsHttpConnection(regionEndPoint.SystemName,
new StaticCredentialsProvider(new AwsCredentials()
{
AccessKey = awsCredentials.GetCredentials().AccessKey,
SecretKey = awsCredentials.GetCredentials().SecretKey,
Token = awsCredentials.GetCredentials().Token
}));
var pool = new SingleNodeConnectionPool(new Uri(endPoint));
ConnectionSettings settings = new ConnectionSettings(pool, conn)
.DisableDirectStreaming()
.InferMappingFor <LogEntry>(m => m.IndexName("logs"));
//.DefaultMappingFor<LogEntry>(m => m.IndexName("logs"));
_esClient = new ElasticClient(settings);
IndexName logIndex = IndexName.From <LogEntry>();
var req = new IndexExistsRequest(logIndex);
var res = _esClient.IndexExists(req);
if (!res.Exists)
{
_esClient.CreateIndex("logs", c => c
.Mappings(md => md.Map <LogEntry>(m => m.AutoMap())));
}
}
internal static IndexDescriptor <T> GetIndex <T>(this ElasticClient client) where T : EsDocument
{
//var col = client.GetIndicesPointingToAlias(aliasName);
var indexDesc = new IndexDescriptor <T>(IndexName.From <T>());
return(indexDesc);
}
public async Task Consume(ConsumeContext <PhotoRemovedEvent> context)
{
await _client.DeleteAsync(new DeleteRequest(IndexName.From <PhotoSearchModel>(), TypeName.From <PhotoSearchModel>(), Id.From(new PhotoSearchModel()
{
OriginalUrl = context.Message.OriginalUrl,
PhotoId = context.Message.PhotoId,
ProductId = context.Message.ProductId
})));
}
public async Task FillElastic(Action <string> output)
{
var node = new Uri(ElasticConstants.Endpoint);
var settings = new ConnectionSettings(node);
var client = new ElasticClient(settings);
var existsResponse = await client.IndexExistsAsync(Indices.Index(ElasticConstants.PlacesCollectionName));
if (!existsResponse.Exists)
{
output?.Invoke("Index does not exist");
var index = settings.DefaultMappingFor <Place>(x => x.IndexName(ElasticConstants.PlacesCollectionName));
client = new ElasticClient(index);
var indexCreate = await client.CreateIndexAsync(IndexName.From <Place>(), i =>
i.Mappings(m =>
m.Map <Place>(mp =>
mp.AutoMap()
.Properties(p =>
p.Text(t => t.Fielddata(true)
.TermVector(TermVectorOption.WithPositionsOffsetsPayloads)
.Name(n => n.Name))
.Text(t => t.Fielddata(true)
.Name(n => n.PlaceId))
.Text(t => t.Fielddata(true)
.TermVector(TermVectorOption.WithPositionsOffsetsPayloads)
.Name(n => n.Vicinity))
.Text(t => t.Fielddata(true)
.Name(n => n.Types)
.Fielddata(true))
)
)));
if (!indexCreate.Acknowledged)
{
output?.Invoke("Error while creating index");
return;
}
output?.Invoke("Index created");
}
var places = await new GooglePlacesService().GetDataAsync();
var bulkResponse = client.Bulk(b =>
{
places.ForEach(p => b.Index <Place>(i => i.Document(p)));
return(b);
});
if (bulkResponse.Errors)
{
foreach (var e in bulkResponse.ItemsWithErrors)
{
output?.Invoke($"{e.Error.Index} - { e.Error.Reason}");
}
}
output?.Invoke("Filled");
}
public async Task <TermVectorsModel> TermVectors(string queryString, double lat, double lng)
{
var client = GetClient();
var searchDescriptor = new SearchDescriptor <Place>();
var dumper = new NestDescriptorDumper(client.RequestResponseSerializer);
#region Search
QueryContainerDescriptor <Place> queryContainer = new QueryContainerDescriptor <Place>();
var query = queryContainer.Bool(b =>
b.Should(
q => q.Match(m => m.Field(f => f.Name).Query(queryString)),
q => q.Match(m => m.Field(f => f.Vicinity).Query(queryString))
)
);
Func <SortDescriptor <Place>, SortDescriptor <Place> > SortByGeo =
(SortDescriptor <Place> s) => s.GeoDistance(g =>
g.Field(f => f.Geometry.Location)
.DistanceType(GeoDistanceType.Arc)
.Unit(DistanceUnit.Kilometers)
.Order(SortOrder.Ascending)
.Points(new GeoLocation(lat, lng)));
Func <SortDescriptor <Place>, IPromise <IList <ISort> > > sort = s => SortByGeo(s.Descending(SortSpecialField.Score).Descending(d => d.Rating));
searchDescriptor
.Index <Place>()
.Query(x => query)
.Sort(sort)
.ScriptFields(x =>
x.ScriptField("distance", s => s.Source($"doc['geometry.location'].arcDistance({lat},{lng})")))
.Take(10)
.Source(true)
;
#endregion
var results = await client.SearchAsync <Place>(searchDescriptor);
var model = ToSearchPlaces(results.Hits).First();
var term = new TermVectorsDescriptor <Place>(IndexName.From <Place>(), TypeName.From <Place>());
term.Index <Place>()
.Id(Id.From(model))
.Fields(tf => tf.Vicinity, tf => tf.Name)
;
var sss = dumper.Dump <TermVectorsDescriptor <Place> >(term);
var termVectorsResult = await client.TermVectorsAsync <Place>(term);
return(ToTermVectorsModel(termVectorsResult, model));
}
[U] public async Task Urls()
{
var project = new Project {
Name = "NEST"
};
await PUT("/project/_create/1")
.Fluent(c => c.Create <object>(new { }, i => i.Index(typeof(Project)).Id(1)))
.Request(c => c.Create(new CreateRequest <object>("project", 1)
{
Document = new { }
}))
.FluentAsync(c => c.CreateAsync <object>(new { }, i => i.Index(typeof(Project)).Id(1)))
.RequestAsync(c => c.CreateAsync(new CreateRequest <object>(IndexName.From <Project>(), 1)
{
Document = new { }
}));
await PUT("/project/_create/NEST")
.Fluent(c => c.CreateDocument(project))
.Fluent(c => c.Create(project, f => f))
.Request(c => c.Create(new CreateRequest <Project>(project)))
.FluentAsync(c => c.CreateDocumentAsync(project))
.FluentAsync(c => c.CreateAsync(project, f => f))
.RequestAsync(c => c.CreateAsync(new CreateRequest <Project>(project)));
await PUT("/project2/_create/NEST")
.Fluent(c => c.Create(project, cc => cc.Index("project2")))
.Request(c => c.Create(new CreateRequest <Project>(project, "project2", "NEST")
{
Document = project
}))
.RequestAsync(c => c.CreateAsync(new CreateRequest <Project>(project, "project2", "NEST")
{
Document = project
}))
.FluentAsync(c => c.CreateAsync(project, cc => cc.Index("project2")));
await PUT("/different-projects/_create/elasticsearch")
.Request(c => c.Create(new CreateRequest <Project>("different-projects", "elasticsearch")
{
Document = project
}))
.Request(c => c.Create(new CreateRequest <Project>(project, "different-projects", "elasticsearch")))
.RequestAsync(c => c.CreateAsync(new CreateRequest <Project>(project, "different-projects", "elasticsearch")))
.RequestAsync(c => c.CreateAsync(new CreateRequest <Project>("different-projects", "elasticsearch")
{
Document = project
}))
;
}
/**[[indices-paths]]
* === Indices paths
*
* Some APIs in Elasticsearch take an index name, a collection of index names,
* or the special `_all` marker (used to specify all indices), in the URI path of the request, to specify the indices that
* the request should execute against.
*
* In NEST, these index names can be specified using the `Indices` type.
*
* ==== Implicit Conversion
*
* To make working with `Indices` easier, several types implicitly convert to it:
*
* - `string`
* - comma separated `string`
* - `string` array
* - a CLR type, <<index-name-inference, where a default index name or index name for the type has been specified on `ConnectionSettings`>>
* - `IndexName`
* - `IndexName` array
*
* Here are some examples of how implicit conversions can be used to specify index names
*/
[U] public void ImplicitConversions()
{
Nest.Indices singleIndexFromString = "name";
Nest.Indices multipleIndicesFromString = "name1, name2";
Nest.Indices multipleIndicesFromStringArray = new [] { "name1", "name2" };
Nest.Indices allFromString = "_all";
Nest.Indices allWithOthersFromString = "_all, name2"; //<1> `_all` will override any specific index names here
Nest.Indices singleIndexFromType = typeof(Project); //<2> The `Project` type has been mapped to a specific index name using <<index-name-type-mapping,`.DefaultMappingFor<Project>`>>
Nest.Indices singleIndexFromIndexName = IndexName.From <Project>();
singleIndexFromString.Match(
all => all.Should().BeNull(),
many => many.Indices.Should().HaveCount(1).And.Contain("name")
);
multipleIndicesFromString.Match(
all => all.Should().BeNull(),
many => many.Indices.Should().HaveCount(2).And.Contain("name2")
);
allFromString.Match(
all => all.Should().NotBeNull(),
many => many.Indices.Should().BeNull()
);
allWithOthersFromString.Match(
all => all.Should().NotBeNull(),
many => many.Indices.Should().BeNull()
);
multipleIndicesFromStringArray.Match(
all => all.Should().BeNull(),
many => many.Indices.Should().HaveCount(2).And.Contain("name2")
);
singleIndexFromType.Match(
all => all.Should().BeNull(),
many => many.Indices.Should().HaveCount(1).And.Contain(typeof(Project))
);
singleIndexFromIndexName.Match(
all => all.Should().BeNull(),
many => many.Indices.Should().HaveCount(1).And.Contain(typeof(Project))
);
}
public static IElasticClient Create()
{
var settings = new ConnectionSettings()
//.ThrowExceptions()
.DisableDirectStreaming()
.DefaultMappingFor <Channel>(m => m.IndexName("chat-channels"))
.DefaultMappingFor <UserAccount>(m => m.IndexName("chat-users"))
.DefaultMappingFor <Sentence>(m => m.IndexName("chat-sentences"));
var client = new ElasticClient(settings);
var existResponse = client.Indices.Exists(IndexName.From <Sentence>());
if (!existResponse.Exists)
{
client.Indices.Create(IndexName.From <Sentence>(), c => c
.Settings(m => m
.Analysis(a => a
.Normalizers(p => p.Custom("lowercase", f => f.Filters("lowercase")))))
.Map <Sentence>(m => m.AutoMap <Sentence>()));
}
existResponse = client.Indices.Exists(IndexName.From <UserAccount>());
if (!existResponse.Exists)
{
client.Indices.Create(IndexName.From <UserAccount>(), c => c
.Settings(m => m
.Analysis(a => a
.Normalizers(p => p.Custom("lowercase", f => f.Filters("lowercase")))))
.Map <UserAccount>(m => m.AutoMap <UserAccount>()));
}
existResponse = client.Indices.Exists(IndexName.From <Channel>());
if (!existResponse.Exists)
{
client.Indices.Create(IndexName.From <Channel>(), c => c
.Settings(m => m
.Analysis(a => a
.Normalizers(p => p.Custom("lowercase", f => f.Filters("lowercase")))))
.Map <Channel>(m => m.AutoMap <Channel>()));
}
return(client);
}
public async Task CleanElastic(Action <string> output)
{
var node = new Uri(ElasticConstants.Endpoint);
var settings = new ConnectionSettings(node);
var client = new ElasticClient(settings);
var existsResponse = await client.IndexExistsAsync(Indices.Index(ElasticConstants.PlacesCollectionName));
if (existsResponse.Exists)
{
var index = settings.DefaultMappingFor <Place>(x => x.IndexName(ElasticConstants.PlacesCollectionName));
client = new ElasticClient(index);
var indexDelete = await client.DeleteIndexAsync(IndexName.From <Place>());
if (!indexDelete.Acknowledged)
{
output?.Invoke("Error while deleting index");
return;
}
output?.Invoke("Index deleted");
}
}
[U] public void UsingWithRequests()
{
var project = new Project {
Name = "hello-world"
};
/** Here we can see and example how DocumentPath helps your describe your requests more tersely */
var request = new IndexRequest <Project>(2)
{
Document = project
};
request = new IndexRequest <Project>(project)
{
};
/** when comparing with the full blown constructor and passing document manually
* DocumentPath<T>'s benefits become apparent.
*/
request = new IndexRequest <Project>(IndexName.From <Project>(), TypeName.From <Project>(), 2)
{
Document = project
};
}
public static IndexName Index <T>(string clusterName) => IndexName.From <T>(clusterName);
[U] public async Task Urls()
{
var name = "name-of-perc";
var index = "indexx";
await POST($"/{index}/.percolator/{name}")
.Fluent(c => c.RegisterPercolator <Project>(name, s => s.Index(index)))
.Request(c => c.RegisterPercolator(new RegisterPercolatorRequest(index, name)))
.FluentAsync(c => c.RegisterPercolatorAsync <Project>(name, s => s.Index(index)))
.RequestAsync(c => c.RegisterPercolatorAsync(new RegisterPercolatorRequest(index, name)))
;
await POST($"/project/.percolator/{name}")
.Fluent(c => c.RegisterPercolator <Project>(name, s => s))
.Request(c => c.RegisterPercolator(new RegisterPercolatorRequest(typeof(Project), name)))
.FluentAsync(c => c.RegisterPercolatorAsync <Project>(name, s => s))
.RequestAsync(c => c.RegisterPercolatorAsync(new RegisterPercolatorRequest(IndexName.From <Project>(), name)))
;
}
private void CreateIndex()
{
var postIndex = IndexName.From <Post>();
#region Default Analyzer
//var aaa = elasticClient.Indices.UpdateSettings(postIndex, p => p
// .IndexSettings(p => p
// .Analysis(p => p
// .Analyzers(p => p
// .UserDefined("default", new SimpleAnalyzer())
// )
// )
// )
//);
//Request:
//PUT http://localhost:9200/my-post-index/_settings?pretty=true&error_trace=true
//{
// "analysis": {
// "analyzer": {
// "default": {
// "type": "simple"
// }
// }
// }
//}
//var bbb = elasticClient.Indices.Create(postIndex, p => p
// .Settings(p => p
// .Analysis(p => p
// .Analyzers(p => p
// .UserDefined("default", new SimpleAnalyzer())
// )
// )
// )
//);
//Request:
//PUT http://localhost:9200/my-post-index?pretty=true&error_trace=true
//{
// "settings": {
// "analysis": {
// "analyzer": {
// "default": {
// "type": "simple"
// }
// }
// }
// }
//}
#endregion
//### Delete
var deleteIndexResponse = elasticClient.Indices.Delete(postIndex);
//### Create Index
var stopwordsPath = Path.Combine(Directory.GetCurrentDirectory(), "stopwords.txt");
//var stopwords = File.ReadAllLines(stopwordsPath).Select(p => p.Trim());
var stopwords = new[] {
"جامع",
"آرام",
"میرود",
"کردند",
"123"
};
#region Persian & Arabic Analyzer/Normalizer
//Persian Chars
//Normalizer: https://github.com/apache/lucenenet/blob/master/src/Lucene.Net.Analysis.Common/Analysis/Fa/PersianNormalizer.cs
//YEH_FARSI (char)1740 == '\u06CC' 'ی'
//YEH (char)1610 == '\u064A' 'ي'
//YEH_BARREE (char)1746 == '\u06D2' 'ے'
//HEH (farsi) (char)1607 == '\u0647' 'ه'
//HEH_YEH (char)1728 == '\u06C0' 'ۀ'
//HEH_GOAL (char)1729 == '\u06C1' 'ہ'
//KEHEH (char)1705 == '\u06A9' 'ک'
//KAF (char)1603 == '\u0643' 'ك'
//HAMZA_ABOVE (char)1620 == '\u0654' 'ٔ'
//ZERO_SPACE (char)8204 == '\u200C' ''
//NORMAL_SPACE (char)32 == '\u0020' ' '
//Persian Fixing
//YEH "\\u064A=>\\u06CC" 'ي' => 'ی'
//YEH_BARREE "\\u06D2=>\\u06CC" 'ے' => 'ی'
//KAF "\\u0643=>\\u06A9" 'ك' => 'ک'
//HEH_YEH "\\u06C0=>\\u0647" 'ۀ' => 'ه'
//HEH_GOAL "\\u06C1=>\\u0647" 'ہ' => 'ه'
//HAMZA_ABOVE REMOVE "\\u0654=>" 'ٔ'
//Arabic Chars (except persians)
//Normalizer: https://github.com/apache/lucenenet/blob/master/src/Lucene.Net.Analysis.Common/Analysis/Ar/ArabicNormalizer.cs
//ALEF (char)1575 == '\u0627' 'ا'
//ALEF_MADDA (char)1570 == '\u0622' 'آ'
//ALEF_HAMZA_ABOVE (char)1571 == '\u0623' 'أ'
//ALEF_HAMZA_BELOW (char)1573 == '\u0625' 'إ'
//DOTLESS_YEH (char)1609 == '\u0649' 'ى'
//TEH_MARBUTA (char)1577 == '\u0629' 'ة'
//TATWEEL (char)1600 == '\u0640' 'ـ' (KhateTire)
//FATHA (char)1614 == '\u064E' 'َ' (Fathe)
//FATHATAN (char)1611 == '\u064B' 'ً' (TanvinFathe)
//DAMMA (char)1615 == '\u064F' 'ُ' (Zamme)
//DAMMATAN (char)1612 == '\u064C' 'ٌ' (TanvinZamme)
//KASRA (char)1616 == '\u0650' 'ِ' (Kasre)
//KASRATAN (char)1613 == '\u064D' 'ٍ' (TanvinKasre)
//SHADDA (char)1617 == '\u0651' 'ّ' (Tashdid)
//SUKUN (char)1618 == '\u0652' 'ْ' (Sokun)
//Arabic Fixing
//ALEF_MADDA "\\u0622=>\\u0627" 'آ' => 'ا'
//ALEF_HAMZA_ABOVE "\\u0623=>\\u0627" 'أ' => 'ا'
//ALEF_HAMZA_BELOW "\\u0625=>\\u0627" 'إ' => 'ا'
//DOTLESS_YEH "\\u0649=>\\u06CC" 'ى' => 'ی' (original normalizer replaces with \u064A 'ي')
//TEH_MARBUTA "\\u0629=>\\u0647" 'ة' => 'ه'
//TATWEEL REMOVE "\\u0640=>" 'ـ' (KhateTire)
//FATHA REMOVE "\\u064E=>" 'َ' (Fathe)
//FATHATAN REMOVE "\\u064B=>" 'ً' (TanvinFathe)
//DAMMA REMOVE "\\u064F=>" 'ُ' (Zamme)
//DAMMATAN REMOVE "\\u064C=>" 'ٌ' (TanvinZamme)
//KASRA REMOVE "\\u0650=>" 'ِ' (Kasre)
//KASRATAN REMOVE "\\u064D=>" 'ٍ' (TanvinKasre)
//SHADDA REMOVE "\\u0651=>" 'ّ' (Tashdid)
//SUKUN REMOVE "\\u0652=>" 'ْ' (Sokun)
//Arab Ameri Chars
//Normalizer: https://github.com/SalmanAA/Lucene.Net.Analysis.Fa/blob/master/Lucene.Net.Analysis.Fa/PersianNormalizer.cs
//Stemer : https://github.com/SalmanAA/Lucene.Net.Analysis.Fa/blob/master/Lucene.Net.Analysis.Fa/PersianStemmer.cs
//HAMZE_JODA (char)1569 == '\u0621' 'ء' (HamzeJoda)
//Arab Ameri Fixing
//HAMZE_JODA REMOVE "\\u0621=>" 'ء' (HamzeJoda)
//var createIndexResponse = elasticClient.Indices.Create(postIndex, p => p
// .Settings(p => p
// .Analysis(p => p
// .CharFilters(p => p
// //https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-mapping-charfilter.html
// .Mapping("mapping_filter", p => p
// .Mappings(
// "\\u200C=>\\u0020"//, //Fix ZERO_SPACE
// //"\\u064A=>\\u06CC", //Fix YEH
// //"\\u06D2=>\\u06CC", //Fix YEH_BARREE
// //"\\u0649=>\\u06CC", //Fix DOTLESS_YEH
// //"\\u0643=>\\u06A9", //Fix KAF
// //"\\u06C0=>\\u0647", //Fix HEH_YEH
// //"\\u06C1=>\\u0647", //Fix HEH_GOAL
// //"\\u0629=>\\u0647", //Fix TEH_MARBUTA
// //"\\u0622=>\\u0627", //Fix ALEF_MADDA
// //"\\u0623=>\\u0627", //Fix ALEF_HAMZA_ABOVE
// //"\\u0625=>\\u0627", //Fix ALEF_HAMZA_BELOW
// //"\\u0654=>", //Remove HAMZA_ABOVE
// //"\\u0640=>", //Remove TATWEEL
// //"\\u064E=>", //Remove FATHA
// //"\\u064B=>", //Remove FATHATAN
// //"\\u064F=>", //Remove DAMMA
// //"\\u064C=>", //Remove DAMMATAN
// //"\\u0650=>", //Remove KASRA
// //"\\u064D=>", //Remove KASRATAN
// //"\\u0651=>", //Remove SHADDA
// //"\\u0652=>" //Remove SUKUN
// //"\\u0621=>" //Remove HAMZE_JODA
// )
// )
// )
// .TokenFilters(p => p
// //https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-stop-tokenfilter.html
// .Stop("persian_stop", p => p
// .StopWords(stopwords)
// .RemoveTrailing()
// //.IgnoreCase()
// )
// .PatternReplace("fix-YEH", p => p.Pattern("\u064A").Replacement("\u06CC"))
// .PatternReplace("fix-YEH_BARREE", p => p.Pattern("\u06D2").Replacement("\u06CC"))
// .PatternReplace("fix-DOTLESS_YEH", p => p.Pattern("\u0649").Replacement("\u06CC"))
// .PatternReplace("fix-KAF", p => p.Pattern("\u0643").Replacement("\u06A9"))
// .PatternReplace("fix-HEH_YEH", p => p.Pattern("\u06C0").Replacement("\u0647"))
// .PatternReplace("fix-HEH_GOAL", p => p.Pattern("\u06C1").Replacement("\u0647"))
// .PatternReplace("fix-TEH_MARBUTA", p => p.Pattern("\u0629").Replacement("\u0647"))
// .PatternReplace("fix-ALEF_MADDA", p => p.Pattern("\u0622").Replacement("\u0627"))
// .PatternReplace("fix-ALEF_HAMZA_ABOVE", p => p.Pattern("\u0623").Replacement("\u0627"))
// .PatternReplace("fix-ALEF_HAMZA_BELOW", p => p.Pattern("\u0625").Replacement("\u0627"))
// .PatternReplace("remove-HAMZA_ABOVE", p => p.Pattern("\u0654").Replacement(string.Empty))
// .PatternReplace("remove-TATWEEL", p => p.Pattern("\u0640").Replacement(string.Empty))
// .PatternReplace("remove-FATHA", p => p.Pattern("\u064E").Replacement(string.Empty))
// .PatternReplace("remove-FATHATAN", p => p.Pattern("\u064B").Replacement(string.Empty))
// .PatternReplace("remove-DAMMA", p => p.Pattern("\u064F").Replacement(string.Empty))
// .PatternReplace("remove-DAMMATAN", p => p.Pattern("\u064C").Replacement(string.Empty))
// .PatternReplace("remove-KASRA", p => p.Pattern("\u0650").Replacement(string.Empty))
// .PatternReplace("remove-KASRATAN", p => p.Pattern("\u064D").Replacement(string.Empty))
// .PatternReplace("remove-SHADDA", p => p.Pattern("\u0651").Replacement(string.Empty))
// .PatternReplace("remove-SUKUN", p => p.Pattern("\u0652").Replacement(string.Empty))
// .PatternReplace("remove-HAMZE_JODA", p => p.Pattern("\u0621").Replacement(string.Empty))
// )
// .Analyzers(p => p
// .Custom("persian_analyzer", p => p
// .Tokenizer("standard") //https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-standard-tokenizer.html
// .CharFilters(
// "html_strip", //https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-htmlstrip-charfilter.html
// "mapping_filter"
// )
// .Filters(
// "lowercase", //https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-lowercase-tokenfilter.html
// "decimal_digit", //https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-decimal-digit-tokenfilter.html
// "persian_stop",
// //"arabic_normalization", //https://lucene.apache.org/core/8_8_0/analyzers-common/org/apache/lucene/analysis/ar/ArabicNormalizer.html
// //"persian_normalization", //https://lucene.apache.org/core/8_8_0/analyzers-common/org/apache/lucene/analysis/fa/PersianNormalizer.html
// "fix-YEH",
// "fix-YEH_BARREE",
// "fix-DOTLESS_YEH",
// "fix-KAF",
// "fix-HEH_YEH",
// "fix-HEH_GOAL",
// "fix-TEH_MARBUTA",
// "fix-ALEF_MADDA",
// "fix-ALEF_HAMZA_ABOVE",
// "fix-ALEF_HAMZA_BELOW",
// "remove-HAMZA_ABOVE",
// "remove-TATWEEL",
// "remove-FATHA",
// "remove-FATHATAN",
// "remove-DAMMA",
// "remove-DAMMATAN",
// "remove-KASRA",
// "remove-KASRATAN",
// "remove-SHADDA",
// "remove-SUKUN",
// "remove-SUKUN",
// "remove-HAMZE_JODA",
// "persian_stop" //remove stopwords before and after normalizations because of ('آ' => 'ا')
// )
// )
// )
// )
// )
// .Map<Post>(p => p
// .AutoMap()
// .Properties(p => p
// //.Number(p => p.Name(p => p.Id))
// .Text(p => p
// .Name(p => p.Title)
// .Analyzer("persian_analyzer")
// )
// .Text(p => p
// .Name(p => p.Body)
// .Analyzer("persian_analyzer")
// //.SearchAnalyzer("persian_analyzer")
// //.Boost(1.5)
// //.Store(true)
// //.Index(true)
// //.Norms(true)
// //.IndexPhrases(true)
// //.IndexOptions(IndexOptions.Offsets)
// //.TermVector(TermVectorOption.WithPositionsOffsetsPayloads)
// )
// )
// )
//);
#endregion
var createIndexResponse = elasticClient.Indices.Create(postIndex, p => p
.Settings(p => p
.Analysis(p => p
.CharFilters(p => p
.Mapping("mapping_filter", p => p
.Mappings(
"\\u200C=>\\u0020" //Fix ZERO_SPACE
)
)
)
.TokenFilters(p => p
.Stop("persian_stop", p => p
.StopWords(new StopWords("_persian_"))
.RemoveTrailing()
)
)
.Analyzers(p => p
.Custom("persian_analyzer", p => p
.Tokenizer("standard")
.CharFilters(
"html_strip",
"mapping_filter"
)
.Filters(
"lowercase",
"decimal_digit",
"arabic_normalization",
"persian_normalization",
"persian_stop"
)
)
)
)
)
.Map <Post>(p => p
.AutoMap()
.Properties(p => p
.Text(p => p
.Name(p => p.Title)
.Analyzer("persian_analyzer")
//.Boost(1.5)
)
.Text(p => p
.Name(p => p.Body)
.Analyzer("persian_analyzer")
//.Boost(1.0)
//.Store(true)
//.Index(true)
//.Norms(true)
//.IndexPhrases(true)
//.IndexOptions(IndexOptions.Offsets)
//.TermVector(TermVectorOption.WithPositionsOffsetsPayloads)
)
)
)
);
//Request:
//PUT /my-post-index?pretty=true&error_trace=true
//{
// "mappings": {
// "properties": {
// "id": {
// "type": "integer"
// },
// "title": {
// "analyzer": "persian_analyzer",
// "boost": 1.5,
// "type": "text"
// },
// "body": {
// "analyzer": "persian_analyzer",
// "boost": 1.0,
// "type": "text"
// }
// }
// },
// "settings": {
// "analysis": {
// "analyzer": {
// "persian_analyzer": {
// "char_filter": [
// "html_strip",
// "mapping_filter"
// ],
// "filter": [
// "lowercase",
// "decimal_digit",
// "arabic_normalization",
// "persian_normalization",
// "persian_stop"
// ],
// "tokenizer": "standard",
// "type": "custom"
// }
// },
// "char_filter": {
// "mapping_filter": {
// "mappings": [
// "\\u200C=>\\u0020"
// ],
// "type": "mapping"
// }
// },
// "filter": {
// "persian_stop": {
// "remove_trailing": true,
// "stopwords": "_persian_",
// "type": "stop"
// }
// }
// }
// }
//}
}
