public void CreateIndex()
{
var client = this._client;
var typeMappingResult = this._client.GetMapping(gm=>gm.Index(ElasticsearchConfiguration.DefaultIndex).Type("elasticsearchprojects"));
var typeMapping = typeMappingResult.Mapping;
typeMapping.Name = "mytype";
var settings = new IndexSettings();
settings.Mappings.Add(typeMapping);
settings.NumberOfReplicas = 1;
settings.NumberOfShards = 5;
settings.Analysis.Analyzers.Add("snowball", new SnowballAnalyzer { Language = "English" });
var indexName = Guid.NewGuid().ToString();
var response = client.CreateIndex(indexName, i=>i.InitializeUsing(settings));
Assert.IsTrue(response.IsValid);
Assert.IsTrue(response.Acknowledged);
var mappingResult = this._client.GetMapping(gm=>gm.Index(indexName).Type("mytype"));
mappingResult.Mapping.Should().NotBeNull();
var deleteResponse = client.DeleteIndex(i=>i.Index(indexName));
Assert.IsTrue(deleteResponse.IsValid);
Assert.IsTrue(deleteResponse.Acknowledged);
}
public void CreateIndexShouldNotThrowNullReference()
{
var settings = new IndexSettings();
settings.Similarity = new SimilaritySettings();
settings.NumberOfReplicas = 1;
settings.NumberOfShards = 5;
settings.Settings.Add("index.refresh_interval", "10s");
settings.Settings.Add("merge.policy.merge_factor", "10");
settings.Settings.Add("search.slowlog.threshold.fetch.warn", "1s");
settings.Analysis.Analyzers.Add(new KeyValuePair<string, AnalyzerBase>("keyword", new KeywordAnalyzer()));
settings.Analysis.Analyzers.Add(new KeyValuePair<string, AnalyzerBase>("simple", new SimpleAnalyzer()));
settings.Mappings.Add(new RootObjectMapping
{
Name = "my_root_object",
Properties = new Dictionary<PropertyNameMarker, IElasticType>
{
{"my_field", new StringMapping() { Name = "my_string_field "}}
}
});
Assert.DoesNotThrow(() =>
{
var idxRsp = this._client.CreateIndex(ElasticsearchConfiguration.NewUniqueIndexName(), i=>i.InitializeUsing(settings));
Assert.IsTrue(idxRsp.IsValid, idxRsp.ConnectionStatus.ToString());
});
}
private void WriteSettings(JsonWriter writer, JsonSerializer serializer, IndexSettings indexSettings)
{
writer.WritePropertyName("settings");
writer.WriteStartObject();
WriteIndexSettings(writer, serializer, indexSettings);
writer.WriteEndObject();
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue,
JsonSerializer serializer)
{
JObject o = JObject.Load(reader);
var result = new IndexSettings();
var dictionary = new Dictionary<string, object>();
serializer.Populate(o.CreateReader(), dictionary);
result.Settings = dictionary;
result._ = DynamicDictionary.Create(dictionary);
foreach (var rootProperty in o.Children<JProperty>())
{
if (rootProperty.Name.Equals("analysis", StringComparison.InvariantCultureIgnoreCase))
{
result.Analysis = serializer.Deserialize<AnalysisSettings>(rootProperty.Value.CreateReader());
result.Settings.Remove(rootProperty.Name);
}
else if (rootProperty.Name.Equals("warmers", StringComparison.InvariantCultureIgnoreCase))
{
foreach (var jWarmer in rootProperty.Value.Children<JProperty>())
{
result.Warmers[jWarmer.Name] = serializer.Deserialize<WarmerMapping>(jWarmer.Value.CreateReader());
}
result.Settings.Remove(rootProperty.Name);
}
else if (rootProperty.Name.Equals("similarity", StringComparison.InvariantCultureIgnoreCase))
{
var baseSimilarity = ((JObject)rootProperty.Value).Property("base");
if (baseSimilarity != null)
{
baseSimilarity.Remove();
result.Similarity = new SimilaritySettings(((JObject)baseSimilarity.Value).Property("type").Value.ToString());
}
else
{
result.Similarity = new SimilaritySettings();
}
foreach (var similarityProperty in rootProperty.Value.Children<JProperty>())
{
var typeProperty = ((JObject)similarityProperty.Value).Property("type");
typeProperty.Remove();
var customSimilarity = new CustomSimilaritySettings(similarityProperty.Name, typeProperty.Value.ToString());
foreach (var similaritySetting in similarityProperty.Value.Children<JProperty>())
{
customSimilarity.SimilarityParameters.Add(similaritySetting.Name, similaritySetting.Value.ToString());
}
result.Similarity.CustomSimilarities.RemoveAll(x => x.Name == customSimilarity.Name);
result.Similarity.CustomSimilarities.Add(customSimilarity);
}
result.Settings.Remove(rootProperty.Name);
}
}
return result;
}
public void CreateIndexMultiFieldMap()
{
var client = this.ConnectedClient;
var typeMapping = new TypeMapping(Guid.NewGuid().ToString("n"));
var property = new TypeMappingProperty
{
Type = "multi_field"
};
var primaryField = new TypeMappingProperty
{
Type = "string",
Index = "not_analyzed"
};
var analyzedField = new TypeMappingProperty
{
Type = "string",
Index = "analyzed"
};
property.Fields = new Dictionary<string, TypeMappingProperty>();
property.Fields.Add("name", primaryField);
property.Fields.Add("name_analyzed", analyzedField);
typeMapping.Properties.Add("name", property);
var settings = new IndexSettings();
settings.Mappings.Add(typeMapping);
settings.NumberOfReplicas = 1;
settings.NumberOfShards = 5;
settings.Analysis.Analyzer.Add("snowball", new SnowballAnalyzerSettings { Language = "English" });
var indexName = Guid.NewGuid().ToString();
var response = client.CreateIndex(indexName, settings);
Assert.IsTrue(response.IsValid);
Assert.IsTrue(response.OK);
Assert.IsNotNull(this.ConnectedClient.GetMapping(indexName, typeMapping.Name));
response = client.DeleteIndex(indexName);
Assert.IsTrue(response.IsValid);
Assert.IsTrue(response.OK);
}
public async Task UpdateAsync([FromBody] IndexSettingsModel model)
{
try
{
IndexSettings indexSettings = Mapper.Map <IndexSettings>(model);
await _indexSettingsService.UpdateAsync(indexSettings);
}
catch (EntityNotFoundException)
{
throw new ValidationApiException(HttpStatusCode.NotFound, "The index settings does not exist");
}
catch (InvalidOperationException exception)
{
throw new ValidationApiException(HttpStatusCode.BadRequest, exception.Message);
}
}
public async Task AddAsync([FromBody] IndexSettingsModel model)
{
try
{
IndexSettings indexSettings = Mapper.Map <IndexSettings>(model);
await _indexSettingsService.AddAsync(indexSettings);
}
catch (EntityAlreadyExistsException)
{
throw new ValidationApiException(HttpStatusCode.Conflict, "The index settings already exists");
}
catch (InvalidOperationException exception)
{
throw new ValidationApiException(HttpStatusCode.BadRequest, exception.Message);
}
}
private void SetupElastic()
{
var settings = new IndexSettings();
settings.NumberOfReplicas = 1;
settings.NumberOfShards = 5;
var response = elasticclient.IndexExists(new IndexExistsRequest("solutionselling.metrics.write"));
if (!response.Exists)
{
var result = elasticclient.CreateIndex("solutionselling.metrics.write");
if (!result.IsValid)
{
Console.WriteLine(result.OriginalException.Message);
}
}
}
public void CreateIndexSettingsForDocument(string index, IndexSettings indexSettings, IndexAliases indexAliases, IndexWarmers indexWarmers)
{
if (_processedItems.Contains("_index" + index))
{
return;
}
_processedItems.Add("_index" + index);
var elasticsearchCrudJsonWriter = new ElasticsearchCrudJsonWriter();
elasticsearchCrudJsonWriter.JsonWriter.WriteStartObject();
CreateIndexSettings(elasticsearchCrudJsonWriter, indexSettings);
indexAliases.WriteJson(elasticsearchCrudJsonWriter);
indexWarmers.WriteJson(elasticsearchCrudJsonWriter);
elasticsearchCrudJsonWriter.JsonWriter.WriteEndObject();
CreateIndexCommand(elasticsearchCrudJsonWriter.GetJsonString(), index);
}
public async Task AddAsync(IndexSettings indexSettings)
{
IndexSettings currentIndexSettings = await GetByIndexAsync(indexSettings.Name);
if (currentIndexSettings != null)
{
throw new EntityAlreadyExistsException();
}
await ValidateAsync(indexSettings);
await _indexSettingsRepository.InsertAsync(indexSettings);
_cache.Set(indexSettings);
_log.InfoWithDetails("Index settings added", indexSettings);
}
public static IElasticClient ConfigureMappings(this IElasticClient client, IConfiguration config)
{
var indexSettings = new IndexSettings()
{
NumberOfReplicas = 0,
NumberOfShards = 1
};
// @TODO:
// - Research mappings between multiple indexes
client.ConfigureEBookMapping(config);
// testing elasticsearch dynamic mapping
client.Indices.Create("users", c => c.Map <User>(m => m.AutoMap()));
return(client);
}
public IndicesResponse CreateIndex(string index, IndexSettings settings)
{
string path = this.CreatePath(index);
string data = JsonConvert.SerializeObject(settings, Formatting.None, this.SerializationSettings);
var status = this.Connection.PostSync(path, data);
var response = new IndicesResponse();
response.ConnectionStatus = status;
try
{
response = JsonConvert.DeserializeObject <IndicesResponse>(status.Result);
response.IsValid = true;
}
catch (Exception e) {
}
return(response);
}
public async Task <UpdateIndexSettingsResponseDto> UpdateIndexSettings <T>(UpdateIndexSettingsRequestDto dto) where T : class, new()
{
try
{
SearchIndex index = client.InitIndex(dto.IndexName);
List <string> CustomRankingOptions = new List <string>();
List <string> SearchableAttributeOptions = new List <string>();
StringBuilder sb = new StringBuilder();
if (dto.IndexOptions != null)
{
#region SetCustomRankingOptions
if (dto.IndexOptions.CustomRankingAttributes != null)
{
foreach (var rankingAttribute in dto.IndexOptions.CustomRankingAttributes)
{
FillCustomRankingOptions <T>(rankingAttribute, sb, CustomRankingOptions);
}
}
#endregion
#region SearchableAttributeOptions
if (dto.IndexOptions.SearchableAttributes != null)
{
foreach (var searchableAttribute in dto.IndexOptions.SearchableAttributes)
{
FillSearchableAttributesOptions <T>(searchableAttribute, sb, SearchableAttributeOptions);
}
}
#endregion
}
IndexSettings settings = new IndexSettings();
settings.CustomRanking = CustomRankingOptions;
settings.SearchableAttributes = SearchableAttributeOptions;
await index.SetSettingsAsync(settings);
return(new UpdateIndexSettingsResponseDto("Index settings updated", true));
}
catch (Exception e)
{
return(new UpdateIndexSettingsResponseDto(e.Message, false));
}
}
public async Task <ResultDetails <string> > CreateIndexAsync(string index, IndexSettings indexSettings, IndexAliases indexAliases, IndexWarmers indexWarmers)
{
if (string.IsNullOrEmpty(index))
{
throw new ElasticsearchCrudException("CreateIndexAsync: index is required");
}
if (indexSettings == null)
{
indexSettings = new IndexSettings {
NumberOfShards = 5, NumberOfReplicas = 1
};
}
if (indexAliases == null)
{
indexAliases = new IndexAliases();
}
if (indexWarmers == null)
{
indexWarmers = new IndexWarmers();
}
_traceProvider.Trace(TraceEventType.Verbose, "{0}: CreateIndexAsync Elasticsearch started", "ElasticsearchContextIndexMapping");
var resultDetails = new ResultDetails <string> {
Status = HttpStatusCode.InternalServerError
};
try
{
MappingUtils.GuardAgainstBadIndexName(index);
var indexMappings = new IndexMappings(_traceProvider, _elasticsearchSerializerConfiguration);
indexMappings.CreateIndexSettingsForDocument(index, indexSettings, indexAliases, indexWarmers);
await indexMappings.Execute(_client, _connectionString, _traceProvider, _cancellationTokenSource);
return(resultDetails);
}
catch (OperationCanceledException oex)
{
_traceProvider.Trace(TraceEventType.Warning, oex, "{1}: CreateIndexAsync Request OperationCanceledException: {0}", oex.Message,
"ElasticsearchContextIndexMapping");
resultDetails.Description = "OperationCanceledException";
return(resultDetails);
}
}
private async Task VerifyIndexExists()
{
var exists = await Client.IndexExistsAsync(_config.IndexName);
if (!exists.Exists)
{
var settings = new IndexSettings();
settings.NumberOfReplicas = 0;
settings.NumberOfShards = 5;
var createRequest = new CreateIndexRequest(new IndexName {
Name = _config.IndexName, Cluster = "fFGNSMh"
});
createRequest.Settings = settings;
await Client.CreateIndexAsync(createRequest);
}
}
public void ShouldBeAbleToSetIndexToReadonly()
{
var settings = new IndexSettings();
settings.Similarity = new SimilaritySettings();
settings.NumberOfReplicas = 1;
settings.NumberOfShards = 5;
settings.Settings.Add("index.blocks.read_only", "true");
var indexName = ElasticsearchConfiguration.NewUniqueIndexName();
var idxRsp = this._client.CreateIndex(indexName, i=>i.InitializeUsing(settings));
Assert.IsTrue(idxRsp.IsValid, idxRsp.ConnectionStatus.ToString());
var getSettingsResponse = this._client.GetIndexSettings(i=>i.Index(indexName));
Assert.IsTrue(getSettingsResponse.IsValid, getSettingsResponse.ConnectionStatus.ToString());
bool readOnly = getSettingsResponse.IndexSettings._.index.blocks.read_only;
readOnly.Should().BeTrue();
}
private void createIndex <T>(string indexName) where T : EsDocument
{
var esClient = Manager.EsClient;
IndexSettings indexSettings = new IndexSettings();
//set up custom analyzers
IndexAnalysers <T>(indexSettings);
IndexState indexConfig = new IndexState
{
Settings = indexSettings
};
esClient.Indices
.Create(indexName, c => c
.InitializeUsing(indexConfig)
.Map(m => m.AutoMap <T>())
);
}
private static void WriteMappings(JsonWriter writer, JsonSerializer serializer, IndexSettings indexSettings)
{
if (indexSettings.Mappings.Count <= 0) return;
var contract = serializer.ContractResolver as SettingsContractResolver;
if (contract == null || contract.ConnectionSettings == null) return;
writer.WritePropertyName("mappings");
serializer.Serialize(
writer,
indexSettings.Mappings.ToDictionary(m =>
{
var name = contract.Infer.PropertyName(m.Name);
if (name.IsNullOrEmpty())
throw new DslException("{0} should have a name!".F(m.GetType()));
return name;
})
);
}
public void CreateIndexMultiFieldMap()
{
var client = this._client;
var typeMapping = new RootObjectMapping();
typeMapping.Name = Guid.NewGuid().ToString("n");
var property = new MultiFieldMapping();
var primaryField = new StringMapping()
{
Index = FieldIndexOption.not_analyzed
};
var analyzedField = new StringMapping()
{
Index = FieldIndexOption.analyzed
};
property.Fields.Add("name", primaryField);
property.Fields.Add("name_analyzed", analyzedField);
typeMapping.Properties = typeMapping.Properties ?? new Dictionary<PropertyNameMarker, IElasticType>();
typeMapping.Properties.Add("name", property);
var settings = new IndexSettings();
settings.Mappings.Add(typeMapping);
settings.NumberOfReplicas = 1;
settings.NumberOfShards = 5;
settings.Analysis.Analyzers.Add("snowball", new SnowballAnalyzer { Language = "English" });
var indexName = Guid.NewGuid().ToString();
var response = client.CreateIndex(indexName, i=>i.InitializeUsing(settings));
Assert.IsTrue(response.IsValid);
Assert.IsTrue(response.Acknowledged);
var inferrer = new ElasticInferrer(this._settings);
var typeName = inferrer.PropertyName(typeMapping.Name);
Assert.IsNotNull(this._client.GetMapping(gm=>gm.Index(indexName).Type(typeName)));
var deleteResponse = client.DeleteIndex(i=>i.Index(indexName));
Assert.IsTrue(deleteResponse.IsValid);
Assert.IsTrue(deleteResponse.Acknowledged);
}
public SettingsOperationResponse UpdateSettings(string index, IndexSettings settings)
{
string path = this.CreatePath(index) + "_settings";
settings.Settings = settings.Settings
.Where(kv => IndexSettings.UpdateWhiteList.Any(p =>
{
return(kv.Key.StartsWith(p));
}
)).ToDictionary(kv => kv.Key, kv => kv.Value);
var sb = new StringBuilder();
var sw = new StringWriter(sb);
using (JsonWriter jsonWriter = new JsonTextWriter(sw))
{
jsonWriter.WriteStartObject();
jsonWriter.WritePropertyName("index");
jsonWriter.WriteStartObject();
foreach (var kv in settings.Settings)
{
jsonWriter.WritePropertyName(kv.Key);
jsonWriter.WriteValue(kv.Value);
}
jsonWriter.WriteEndObject();
}
string data = sb.ToString();
var status = this.Connection.PutSync(path, data);
var r = new SettingsOperationResponse();
try
{
r = JsonConvert.DeserializeObject <SettingsOperationResponse>(status.Result);
}
catch
{
}
r.IsValid = status.Success;
r.ConnectionStatus = status;
return(r);
}
public async Task UpdateAsync(IndexSettings indexSettings)
{
IndexSettings currentIndexSettings = await GetByIndexAsync(indexSettings.Name);
if (currentIndexSettings == null)
{
throw new EntityNotFoundException();
}
await ValidateAsync(indexSettings);
currentIndexSettings.Update(indexSettings);
await _indexSettingsRepository.UpdateAsync(currentIndexSettings);
_cache.Set(currentIndexSettings);
_log.InfoWithDetails("Index settings updated", currentIndexSettings);
}
public void PrefixQueryShouldReturnHits()
{
var settings = new IndexSettings();
settings.NumberOfShards = 2;
settings.Settings.Add("merge.policy.merge_factor", "5");
settings.Settings.Add("search.slowlog.threshold.fetch.warn", "1s");
var index = "issue960";
if (this.Client.IndexExists(p => p.Index(index)).Exists)
{
this.Client.DeleteIndex(p => p.Index(index));
}
var createResponse = this.Client.CreateIndex(index, c => c
.AddMapping<Document>(m => m
.Properties(p => p
.String(s => s.Name(d => d.DocNumber).Analyzer("keyword")
)
)
)
);
createResponse.IsValid.Should().BeTrue();
var document = new Document { DocNumber = "1.2345+t" };
var indexResponse = this.Client.Index<Document>(document, i => i
.Index(index)
.Refresh()
);
indexResponse.IsValid.Should().BeTrue();
var searchResponse = this.Client.Search<Document>(s => s
.Index(index)
.Type("document")
.Query(q => q
.Prefix(doc => doc.DocNumber, "1.2345+")
)
);
searchResponse.IsValid.Should().BeTrue();
searchResponse.Hits.Count().ShouldBeEquivalentTo(1);
}
public static IndexSettings GetDefaultIndexSettings()
{
var settings = new IndexSettings();
settings.AttributesToIndex = IndexingSettings.IncludedFields
.Where(f => f.Indexed)
.Select(f => f.FieldName)
.ToList();
settings.AttributesForFaceting = IndexingSettings.IncludedFields
.Where(f => f.FieldType == FieldType.Facet || f.FieldType == FieldType.Boolean)
.Select(f => f.FieldName)
.ToList();
settings.AttributesForFaceting.AddRange(IndexingSettings.IncludedFields
.Where(f => f.FieldType == FieldType.Facet)
.Select(f => f.FieldName + "_ids"));
return(settings);
}
public void CreateIndex()
{
var indexSettings = new IndexSettings {
NumberOfReplicas = 1, NumberOfShards = 1
};
var indexConfig = new IndexState
{
Settings = indexSettings
};
//if (!_client.IndexExists(_elasticIndex).Exists)
//{
// var result = _client.CreateIndex(_elasticIndex, ci => ci
// .InitializeUsing(indexConfig)
// .Mappings(m => m.Map<T>(mp => mp.AutoMap()))
// );
//}
}
public async Task UpdateAsync(Index index)
{
if(!index.ValidateValue())
throw new InvalidOperationException("Invalid index value");
if(!index.ValidateWeights())
throw new InvalidOperationException("Invalid index weights");
IndexSettings indexSettings = await _indexSettingsService.GetByIndexAsync(index.Name);
if (indexSettings == null)
throw new InvalidOperationException("Index settings not found");
IndexPrice indexPrice = await GetByIndexAsync(index.Name);
if (indexPrice == null)
{
indexPrice = IndexPrice.Init(index.Name, index.Value, index.Timestamp, index.Weights);
await _indexPriceRepository.InsertAsync(indexPrice);
_log.InfoWithDetails("The index price initialized", indexPrice);
}
else
{
IndexSettlementPrice indexSettlementPrice = IndexSettlementPriceCalculator.Calculate(
index.Value, indexPrice.Value, indexSettings.Alpha, indexPrice.K, indexPrice.Price,
index.Timestamp, indexPrice.Timestamp, indexSettings.TrackingFee, indexSettings.PerformanceFee);
indexPrice.Update(index.Value, index.Timestamp, indexSettlementPrice.Price, indexSettlementPrice.K,
indexSettlementPrice.R, indexSettlementPrice.Delta, index.Weights);
await _indexPriceRepository.UpdateAsync(indexPrice);
_log.InfoWithDetails("The index price calculated", new
{
indexPrice,
indexSettings
});
}
_cache.Set(indexPrice);
}
private async Task ReleaseClientReservedFundsAsync(Settlement settlement)
{
if (settlement.Status != SettlementStatus.Reserved)
{
settlement.Status = SettlementStatus.Rejected;
await _settlementRepository.UpdateAsync(settlement);
return;
}
IndexSettings indexSettings = await _indexSettingsService.GetByIndexAsync(settlement.IndexName);
// TODO: Use internal asset
AssetSettings assetSettings = (await _instrumentService.GetAssetsAsync())
.Single(o => o.Exchange == ExchangeNames.Lykke && o.AssetId == indexSettings.AssetId);
try
{
await _settlementTransferService.ReleaseClientReservedFundsAsync(settlement.WalletId,
assetSettings.Asset, settlement.Amount, settlement.ClientId, settlement.Id);
settlement.Status = SettlementStatus.Rejected;
_log.InfoWithDetails("Reserved client funds released",
new { SettlementId = settlement.Id, indexSettings.AssetId, settlement.Amount, settlement.WalletId });
}
catch (NotEnoughFundsException)
{
settlement.Error = SettlementError.NotEnoughFunds;
_log.WarningWithDetails("Not enough reserved client funds to release",
new { SettlementId = settlement.Id, indexSettings.AssetId, settlement.Amount, settlement.WalletId });
}
catch (Exception exception)
{
settlement.Error = SettlementError.Unknown;
_log.ErrorWithDetails(exception, "An error occurred while releasing reserved client funds",
new { SettlementId = settlement.Id, indexSettings.AssetId, settlement.Amount, settlement.WalletId });
}
await _settlementRepository.UpdateAsync(settlement);
}
private async Task CompleteSettlementAsync(Settlement settlement)
{
try
{
IndexSettings indexSettings = await _indexSettingsService.GetByIndexAsync(settlement.IndexName);
await _tokenService.CloseAsync(indexSettings.AssetId, settlement.Amount, settlement.Price);
settlement.Status = SettlementStatus.Completed;
}
catch (Exception exception)
{
settlement.Error = SettlementError.Unknown;
_log.ErrorWithDetails(exception, "An error occurred while completing settlement", settlement);
}
await _settlementRepository.UpdateAsync(settlement);
}
public void TestLegacySettings()
{
string json =
"{ \"attributesToIndex\":[\"attr1\", \"attr2\"],\"numericAttributesToIndex\": [\"attr1\", \"attr2\"],\"slaves\":[\"index1\", \"index2\"]}";
IndexSettings settings = JsonConvert.DeserializeObject <IndexSettings>(json);
Assert.IsNotNull(settings.Replicas);
Assert.True(settings.Replicas.Contains("index1"));
Assert.True(settings.Replicas.Contains("index2"));
Assert.IsNotNull(settings.SearchableAttributes);
Assert.True(settings.SearchableAttributes.Contains("attr1"));
Assert.True(settings.SearchableAttributes.Contains("attr2"));
Assert.IsNotNull(settings.NumericAttributesForFiltering);
Assert.True(settings.NumericAttributesForFiltering.Contains("attr1"));
Assert.True(settings.NumericAttributesForFiltering.Contains("attr2"));
}
private async Task TransferClientReservedFundsAsync(Settlement settlement)
{
IndexSettings indexSettings = await _indexSettingsService.GetByIndexAsync(settlement.IndexName);
// TODO: Use internal asset
AssetSettings assetSettings = (await _instrumentService.GetAssetsAsync())
.Single(o => o.Exchange == ExchangeNames.Lykke && o.AssetId == indexSettings.AssetId);
try
{
string transactionId = await _settlementTransferService.TransferClientReservedFundsAsync(
assetSettings.Asset, settlement.Amount, settlement.ClientId, settlement.Id);
settlement.TransactionId = transactionId;
settlement.Status = SettlementStatus.Transferred;
_log.InfoWithDetails("Reserved funds transferred to main wallet",
new
{
SettlementId = settlement.Id,
indexSettings.AssetId,
settlement.Amount,
settlement.WalletId,
transactionId
});
}
catch (NotEnoughFundsException)
{
settlement.Error = SettlementError.NotEnoughFunds;
_log.WarningWithDetails("Not enough reserved funds to transfer to main wallet",
new { SettlementId = settlement.Id, indexSettings.AssetId, settlement.Amount, settlement.WalletId });
}
catch (Exception exception)
{
settlement.Error = SettlementError.Unknown;
_log.ErrorWithDetails(exception, "An error occurred while transferring reserved funds to main wallet",
new { SettlementId = settlement.Id, indexSettings.AssetId, settlement.Amount, settlement.WalletId });
}
await _settlementRepository.UpdateAsync(settlement);
}
private void btnCreateIndex_Click(object sender, EventArgs e)
{
var settings = new IndexSettings {
NumberOfReplicas = 1, NumberOfShards = 2
};
var indexConfig = new IndexState
{
Settings = settings
};
if (!m_EsClient.IndexExists(txtIndexName.Text).Exists)
{
var _response = m_EsClient.CreateIndex(txtIndexName.Text, c => c
.InitializeUsing(indexConfig)
.Mappings(m => m.Map <Post>(mp => mp.AutoMap()))
);
}
}
public static void CreateGenreIndex()
{
string indexName = "genres";
string aliasName = "my_genre";
var indexSettings = new IndexSettings();
indexSettings.NumberOfReplicas = 1;
indexSettings.NumberOfShards = 3;
var createIndexDescriptor = new CreateIndexDescriptor(indexName)
.Mappings(ms => ms.Map <Genre>(m => m.AutoMap()))
.InitializeUsing(new IndexState()
{
Settings = indexSettings
})
.Aliases(a => a.Alias(aliasName));
ElasticsearchHelper.EsClient().CreateIndex(createIndexDescriptor);
}
static void Main(string[] args)
{
EsNode = new Uri("http://localhost:9200/");
EsConfig = new ConnectionSettings(EsNode).DefaultIndex("employee");
EsClient = new ElasticClient(EsConfig);
var settings = new IndexSettings {
NumberOfReplicas = 1, NumberOfShards = 2
};
var indexConfig = new IndexState
{
Settings = settings
};
if (!EsClient.IndexExists("employee").Exists)
{
EsClient.CreateIndex("employee", c => c
.InitializeUsing(indexConfig)
.Mappings(m => m.Map <Employee>(mp => mp.AutoMap()))
);
}
//InsertDocument();
//InsertBulkDocument();
//PopulateDocument();
//UpdateDocument();
//UpdateDocumentWithNewField();
//GetDocument();
//SearchDocumentMethod1();
//SearchDocumentMethod2();
//SearchDocumentUsingOROperator();
//SearchDocumentUsingANDOperator();
//SearchDocumentUsingNOTOperator();
//SearchDocumentUsingOperatorOverloading();
//SearchDocumentUsingFilter();
//SearchDocumentComplex1();
//SearchDocumentComplex2();
//SearchDocumentComplex3();
//DeleteIndex();
//DeleteDocument();
}
public void CreateIndexMultiFieldMap()
{
var client = this._client;
var typeMapping = new RootObjectMapping();
typeMapping.Name = Guid.NewGuid().ToString("n");
var property = new MultiFieldMapping();
var primaryField = new StringMapping()
{
Index = FieldIndexOption.not_analyzed
};
var analyzedField = new StringMapping()
{
Index = FieldIndexOption.analyzed
};
property.Fields.Add("name", primaryField);
property.Fields.Add("name_analyzed", analyzedField);
typeMapping.Properties = typeMapping.Properties ?? new Dictionary<string, IElasticType>();
typeMapping.Properties.Add("name", property);
var settings = new IndexSettings();
settings.Mappings.Add(typeMapping);
settings.NumberOfReplicas = 1;
settings.NumberOfShards = 5;
settings.Analysis.Analyzer.Add("snowball", new SnowballAnalyzerSettings { Language = "English" });
var indexName = Guid.NewGuid().ToString();
var response = client.CreateIndex(indexName, settings);
Assert.IsTrue(response.IsValid);
Assert.IsTrue(response.OK);
Assert.IsNotNull(this._client.GetMapping(indexName, typeMapping.Name));
response = client.DeleteIndex(indexName);
Assert.IsTrue(response.IsValid);
Assert.IsTrue(response.OK);
}
// check out this handy site
// https://hassantariqblog.wordpress.com/category/back-end-stuff/elastic-search/
// and the .net NEST lib is in. https://github.com/elastic/elasticsearch-net
private void TestElasticSearch()
{
node = new Uri("https://search-testtbk-vibxsyhb7jpdbcth5337jovgl4.us-east-1.es.amazonaws.com");
settings = new ConnectionSettings(node);
settings.DefaultIndex("contentdb");
client = new ElasticClient(settings);
// Mapping for indexes
var indexSettings = new IndexSettings();
indexSettings.NumberOfReplicas = 1;
indexSettings.NumberOfShards = 1;
// Create properties from the Post class
// https://www.elastic.co/guide/en/elasticsearch/client/net-api/current/fluent-mapping.html
CreateAndLoad();
TestUpsert();
//TestPushToESViaSNS();
}
public void AddtoDatabase(Region region)
{
pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
connectionSettings = new ConnectionSettings(pool).DefaultMappingFor <Region>(m => m.IndexName("region"));
connectionSettings.DefaultIndex("RegionID");
client = new ElasticClient(connectionSettings);
IndexSettings settings = new IndexSettings {
NumberOfReplicas = 1, NumberOfShards = 1
};
IndexState indexConfig = new IndexState {
Settings = settings
};
//Console.Read();
var result = client.Bulk(b => b.Index <Region>(i => i.Document(region))
.Create <Region>(c => c.Document(region))
);
}
static void CreateIndex()
{
var settings = new IndexSettings {
NumberOfReplicas = 1, NumberOfShards = 2
};
var indexConfig = new IndexState
{
Settings = settings
};
if (!EsClient.IndexExists("QuotationLine").Exists)
{
EsClient.CreateIndex("QuotationLine", c => c
.InitializeUsing(indexConfig)
.Mappings(m => m.Map <QuotationLine>(mp => mp.AutoMap())));
}
}
// check out this handy site
// https://hassantariqblog.wordpress.com/category/back-end-stuff/elastic-search/
// and the .net NEST lib is in. https://github.com/elastic/elasticsearch-net
private void TestElasticSearch()
{
node = new Uri("http://localhost:9200");
settings = new ConnectionSettings(node);
settings.DefaultIndex("contentidx");
client = new ElasticClient(settings);
// Mapping for indexes
var indexSettings = new IndexSettings();
indexSettings.NumberOfReplicas = 1;
indexSettings.NumberOfShards = 1;
// Create properties from the Post class
// https://www.elastic.co/guide/en/elasticsearch/client/net-api/current/fluent-mapping.html
// after creating this, you can issue
// GET /myindex in the command area.
// https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-get-index.html
if (client.IndexExists("contentidx").Exists)
{
TestDeleteIndex();
}
var createIndexResponse = client.CreateIndex("contentidx", c => c
.Mappings(ms => ms
.Map <Content>(m => m
.AutoMap(typeof(Content))
)
)
);
// todo, check createIndexResponse
TestInsert();
TestTermQuery();
TestMatchPhrase();
TestFilter();
}
private IReadOnlyCollection <LimitOrder> CreateLimitOrders(IndexSettings indexSettings,
AssetPairSettings assetPairSettings, decimal sellPrice, decimal buyPrice)
{
var limitOrders = new List <LimitOrder>();
string walletId = _settingsService.GetWalletId();
decimal sellVolume = Math.Round(indexSettings.SellVolume / indexSettings.SellLimitOrdersCount,
assetPairSettings.VolumeAccuracy);
if (sellVolume >= assetPairSettings.MinVolume)
{
for (int i = 0; i < indexSettings.SellLimitOrdersCount; i++)
{
limitOrders.Add(LimitOrder.CreateSell(walletId, sellPrice, sellVolume));
}
}
else
{
limitOrders.Add(LimitOrder.CreateSell(walletId, sellPrice,
Math.Round(indexSettings.SellVolume, assetPairSettings.VolumeAccuracy)));
}
decimal buyVolume = Math.Round(indexSettings.BuyVolume / indexSettings.BuyLimitOrdersCount,
assetPairSettings.VolumeAccuracy);
if (buyVolume >= assetPairSettings.MinVolume)
{
for (int i = 0; i < indexSettings.BuyLimitOrdersCount; i++)
{
limitOrders.Add(LimitOrder.CreateBuy(walletId, buyPrice, buyVolume));
}
}
else
{
limitOrders.Add(LimitOrder.CreateBuy(walletId, buyPrice,
Math.Round(indexSettings.BuyVolume, assetPairSettings.VolumeAccuracy)));
}
return(limitOrders);
}
public void CreateIndex(string indexName)
{
if (!_elasticClient.IndexExists(indexName).Exists)
{
var indexSettings = new IndexSettings();
indexSettings.NumberOfReplicas = 1;
indexSettings.NumberOfShards = 3;
var createIndexDescriptor = new CreateIndexDescriptor(indexName)
.Mappings(ms => ms
.Map <LogDto>(m => m
.AutoMap()))
.InitializeUsing(new IndexState()
{
Settings = indexSettings
})
.Aliases(x => x.Alias(indexName));
ICreateIndexResponse createIndexResponse = _elasticClient.CreateIndex(createIndexDescriptor);
}
}
public static ElasticClient EsClient()
{
ConnectionSettings connectionSettings;
ElasticClient elasticClient;
//Connection string for Elasticsearch
connectionSettings = new ConnectionSettings(new Uri("http://localhost:9200/")); //local PC
elasticClient = new ElasticClient(connectionSettings);
var settings = new IndexSettings {
NumberOfReplicas = 1, NumberOfShards = 2
};
var indexConfig = new IndexState
{
Settings = settings
};
return(elasticClient);
}
private void WriteIndexSettings(JsonWriter writer, JsonSerializer serializer, IndexSettings indexSettings)
{
writer.WritePropertyName("index");
writer.WriteStartObject();
if (indexSettings.Settings.HasAny())
{
foreach (var kv in indexSettings.Settings)
{
writer.WritePropertyName(kv.Key);
if (kv.Value is JObject)
this.WriteSettingObject(writer, kv.Value as JObject);
else
writer.WriteValue(kv.Value);
}
}
if (
indexSettings.Analysis.Analyzers.Count > 0
|| indexSettings.Analysis.TokenFilters.Count > 0
|| indexSettings.Analysis.Tokenizers.Count > 0
|| indexSettings.Analysis.CharFilters.Count > 0
)
{
writer.WritePropertyName("analysis");
serializer.Serialize(writer, indexSettings.Analysis);
}
if (
indexSettings.Similarity.CustomSimilarities.Count > 0
|| !string.IsNullOrEmpty(indexSettings.Similarity.Default)
)
{
writer.WritePropertyName("similarity");
serializer.Serialize(writer, indexSettings.Similarity);
}
writer.WriteEndObject();
}
public void CreateIndex()
{
var client = this._client;
var typeMapping = this._client.GetMapping(ElasticsearchConfiguration.DefaultIndex, "elasticsearchprojects");
typeMapping.TypeNameMarker = "mytype";
var settings = new IndexSettings();
settings.Mappings.Add(typeMapping);
settings.NumberOfReplicas = 1;
settings.NumberOfShards = 5;
settings.Analysis.Analyzers.Add("snowball", new SnowballAnalyzer { Language = "English" });
var indexName = Guid.NewGuid().ToString();
var response = client.CreateIndex(indexName, settings);
Assert.IsTrue(response.IsValid);
Assert.IsTrue(response.OK);
Assert.IsNotNull(this._client.GetMapping(indexName, "mytype"));
var deleteResponse = client.DeleteIndex(indexName);
Assert.IsTrue(deleteResponse.IsValid);
Assert.IsTrue(deleteResponse.OK);
}
public void GetIndexSettingsComplex()
{
var index = Guid.NewGuid().ToString();
var settings = new IndexSettings();
settings.NumberOfReplicas = 4;
settings.NumberOfShards = 8;
settings.Analysis.Analyzers.Add("snowball", new SnowballAnalyzer { Language = "English" });
settings.Analysis.Analyzers.Add("standard", new StandardAnalyzer { StopWords = new[]{"word1", "word2"}});
settings.Analysis.Analyzers.Add("swedishlanguage", new LanguageAnalyzer(Language.Swedish) { StopWords = new[] { "word1", "word2" }, StemExclusionList = new[] { "stem1", "stem2" } });
settings.Analysis.CharFilters.Add("char1", new HtmlStripCharFilter());
settings.Analysis.CharFilters.Add("char2", new MappingCharFilter{ Mappings = new []{"ph=>f", "qu=>q"}});
settings.Analysis.TokenFilters.Add("tokenfilter1", new EdgeNGramTokenFilter());
settings.Analysis.TokenFilters.Add("tokenfilter2", new SnowballTokenFilter());
settings.Analysis.Tokenizers.Add("token1", new KeywordTokenizer());
settings.Analysis.Tokenizers.Add("token2", new PathHierarchyTokenizer());
settings.Similarity = new SimilaritySettings();
var dfr = new CustomSimilaritySettings("test1", "DFR");
dfr.SimilarityParameters.Add("basic_model", "g");
dfr.SimilarityParameters.Add("after_effect", "l");
dfr.SimilarityParameters.Add("normalization", "h2");
dfr.SimilarityParameters.Add("normalization.h2.c", 3);
settings.Similarity.CustomSimilarities.Add(dfr);
var ib = new CustomSimilaritySettings("test2", "IB");
ib.SimilarityParameters.Add("distribution", "spl");
ib.SimilarityParameters.Add("lambda", "ttf");
ib.SimilarityParameters.Add("normalization", "h1");
settings.Similarity.CustomSimilarities.Add(ib);
var typeMapping = this._client.GetMapping(ElasticsearchConfiguration.DefaultIndex, "elasticsearchprojects");
typeMapping.TypeNameMarker = index;
settings.Mappings.Add(typeMapping);
settings.Add("merge.policy.merge_factor", "10");
var createResponse = this._client.CreateIndex(index, settings);
var r = this._client.GetIndexSettings(index);
Assert.True(r.IsValid);
Assert.NotNull(r.Settings);
Assert.AreEqual(r.Settings.NumberOfReplicas, 4);
Assert.AreEqual(r.Settings.NumberOfShards, 8);
Assert.Greater(r.Settings.Count(), 0);
Assert.True(r.Settings.ContainsKey("merge.policy.merge_factor"));
Assert.AreEqual("10", r.Settings["merge.policy.merge_factor"]);
Assert.AreEqual(3, r.Settings.Analysis.Analyzers.Count);
{ // assert analyzers
Assert.True(r.Settings.Analysis.Analyzers.ContainsKey("snowball"));
var snoballAnalyser = r.Settings.Analysis.Analyzers["snowball"] as SnowballAnalyzer;
Assert.NotNull(snoballAnalyser);
Assert.AreEqual("English", snoballAnalyser.Language);
Assert.True(r.Settings.Analysis.Analyzers.ContainsKey("standard"));
var standardAnalyser = r.Settings.Analysis.Analyzers["standard"] as StandardAnalyzer;
Assert.NotNull(standardAnalyser);
Assert.NotNull(standardAnalyser.StopWords);
Assert.AreEqual(2, standardAnalyser.StopWords.Count());
Assert.True(standardAnalyser.StopWords.Contains("word1"));
Assert.True(standardAnalyser.StopWords.Contains("word2"));
Assert.True(r.Settings.Analysis.Analyzers.ContainsKey("swedishlanguage"));
var languageAnalyser = r.Settings.Analysis.Analyzers["swedishlanguage"] as LanguageAnalyzer;
Assert.NotNull(languageAnalyser);
Assert.AreEqual(Language.Swedish.ToString().ToLower(), languageAnalyser.Type);
Assert.NotNull(languageAnalyser.StopWords);
Assert.AreEqual(2, languageAnalyser.StopWords.Count());
Assert.True(languageAnalyser.StopWords.Contains("word1"));
Assert.True(languageAnalyser.StopWords.Contains("word2"));
Assert.AreEqual(2, languageAnalyser.StemExclusionList.Count());
Assert.True(languageAnalyser.StemExclusionList.Contains("stem1"));
Assert.True(languageAnalyser.StemExclusionList.Contains("stem2"));
}
Assert.AreEqual(2, r.Settings.Analysis.CharFilters.Count);
{ // assert char filters
Assert.True(r.Settings.Analysis.CharFilters.ContainsKey("char1"));
var filter1 = r.Settings.Analysis.CharFilters["char1"] as HtmlStripCharFilter;
Assert.NotNull(filter1);
Assert.True(r.Settings.Analysis.CharFilters.ContainsKey("char2"));
var filter2 = r.Settings.Analysis.CharFilters["char2"] as MappingCharFilter;
Assert.NotNull(filter2);
Assert.AreEqual(2, filter2.Mappings.Count());
Assert.True(filter2.Mappings.Contains("ph=>f"));
Assert.True(filter2.Mappings.Contains("qu=>q"));
}
Assert.AreEqual(2, r.Settings.Analysis.TokenFilters.Count);
{ // assert token filters
Assert.True(r.Settings.Analysis.TokenFilters.ContainsKey("tokenfilter1"));
var filter1 = r.Settings.Analysis.TokenFilters["tokenfilter1"] as EdgeNGramTokenFilter;
Assert.NotNull(filter1);
Assert.True(r.Settings.Analysis.TokenFilters.ContainsKey("tokenfilter2"));
var filter2 = r.Settings.Analysis.TokenFilters["tokenfilter2"] as SnowballTokenFilter;
Assert.NotNull(filter2);
}
Assert.AreEqual(2, r.Settings.Analysis.Tokenizers.Count);
{ // assert tokenizers
Assert.True(r.Settings.Analysis.Tokenizers.ContainsKey("token1"));
var tokenizer1 = r.Settings.Analysis.Tokenizers["token1"] as KeywordTokenizer;
Assert.NotNull(tokenizer1);
Assert.True(r.Settings.Analysis.Tokenizers.ContainsKey("token2"));
var tokenizer2 = r.Settings.Analysis.Tokenizers["token2"] as PathHierarchyTokenizer;
Assert.NotNull(tokenizer2);
}
Assert.NotNull(r.Settings.Similarity);
Assert.NotNull(r.Settings.Similarity.CustomSimilarities);
Assert.AreEqual(2, r.Settings.Similarity.CustomSimilarities.Count);
{ // assert similarity
var similarity1 = r.Settings.Similarity.CustomSimilarities.FirstOrDefault(x => x.Name.Equals("test1", StringComparison.InvariantCultureIgnoreCase));
Assert.NotNull(similarity1);
Assert.AreEqual("DFR", similarity1.Type);
Assert.AreEqual(4, similarity1.SimilarityParameters.Count);
Assert.True(similarity1.SimilarityParameters.Any(x => x.Key.Equals("basic_model") && x.Value.ToString().Equals("g")));
Assert.True(similarity1.SimilarityParameters.Any(x => x.Key.Equals("after_effect") && x.Value.ToString().Equals("l")));
Assert.True(similarity1.SimilarityParameters.Any(x => x.Key.Equals("normalization") && x.Value.ToString().Equals("h2")));
Assert.True(similarity1.SimilarityParameters.Any(x => x.Key.Equals("normalization.h2.c") && x.Value.ToString().Equals("3")));
var similarity2 = r.Settings.Similarity.CustomSimilarities.FirstOrDefault(x => x.Name.Equals("test2", StringComparison.InvariantCultureIgnoreCase));
Assert.NotNull(similarity2);
Assert.AreEqual("IB", similarity2.Type);
Assert.AreEqual(3, similarity2.SimilarityParameters.Count);
Assert.True(similarity2.SimilarityParameters.Any(x => x.Key.Equals("distribution") && x.Value.ToString().Equals("spl")));
Assert.True(similarity2.SimilarityParameters.Any(x => x.Key.Equals("lambda") && x.Value.ToString().Equals("ttf")));
Assert.True(similarity2.SimilarityParameters.Any(x => x.Key.Equals("normalization") && x.Value.ToString().Equals("h1")));
}
this._client.DeleteIndex(index);
}
private static void WriteAliases(JsonWriter writer, JsonSerializer serializer, IndexSettings indexSettings)
{
if (indexSettings.Aliases == null || indexSettings.Aliases.Count <= 0) return;
writer.WritePropertyName("aliases");
serializer.Serialize(writer, indexSettings.Aliases);
}
public void GetIndexSettingsComplex()
{
var index = Guid.NewGuid().ToString();
var settings = new IndexSettings();
settings.NumberOfReplicas = 4;
settings.NumberOfShards = 8;
settings.Analysis.Analyzers.Add("snowball", new SnowballAnalyzer { Language = "English" });
var typeMapping = this._client.GetMapping(ElasticsearchConfiguration.DefaultIndex, "elasticsearchprojects");
typeMapping.Name = index;
settings.Mappings.Add(typeMapping);
settings.Add("merge.policy.merge_factor", "10");
var createResponse = this._client.CreateIndex(index, settings);
var r = this._client.GetIndexSettings(index);
Assert.True(r.IsValid);
Assert.NotNull(r.Settings);
Assert.AreEqual(r.Settings.NumberOfReplicas, 4);
Assert.AreEqual(r.Settings.NumberOfShards, 8);
Assert.Greater(r.Settings.Count(), 0);
Assert.True(r.Settings.ContainsKey("merge.policy.merge_factor"));
this._client.DeleteIndex(index);
}
private static void WriteWarmers(JsonWriter writer, JsonSerializer serializer, IndexSettings indexSettings)
{
if (indexSettings.Warmers.Count <= 0) return;
writer.WritePropertyName("warmers");
serializer.Serialize(writer, indexSettings.Warmers);
}
private static void WriteSimilarityModuleSettings(JsonWriter writer, IndexSettings indexSettings)
{
if (indexSettings.Similarity == null) return;
writer.WritePropertyName("similarity");
writer.WriteStartObject();
if (!string.IsNullOrEmpty(indexSettings.Similarity.BaseSimilarity))
{
writer.WritePropertyName("base");
writer.WriteStartObject();
writer.WritePropertyName("type");
writer.WriteValue(indexSettings.Similarity.BaseSimilarity);
writer.WriteEndObject();
}
if (indexSettings.Similarity.CustomSimilarities != null)
{
foreach (var customSimilarity in indexSettings.Similarity.CustomSimilarities)
{
writer.WritePropertyName(customSimilarity.Name);
writer.WriteStartObject();
writer.WritePropertyName("type");
writer.WriteValue(customSimilarity.Type);
if (customSimilarity.SimilarityParameters.HasAny())
{
foreach (var kv in customSimilarity.SimilarityParameters)
{
writer.WritePropertyName(kv.Key);
writer.WriteValue(kv.Value);
}
}
writer.WriteEndObject();
}
}
writer.WriteEndObject();
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue,
JsonSerializer serializer)
{
var result = new IndexSettings();
if (reader.TokenType != JsonToken.StartObject) return result;
var jsonObject = JObject.Load(reader);
if (jsonObject["settings"] != null && jsonObject["settings"]["index"] != null)
{
var settings = jsonObject["settings"]["index"];
var dictionary = new Dictionary<string, object>();
serializer.Populate(settings.CreateReader(), dictionary);
result.Settings = dictionary;
result.AsExpando = DynamicDictionary.Create(dictionary);
foreach (var rootProperty in settings.Children<JProperty>())
{
if (rootProperty.Name.Equals("analysis", StringComparison.InvariantCultureIgnoreCase))
{
result.Analysis = serializer.Deserialize<AnalysisSettings>(rootProperty.Value.CreateReader());
result.Settings.Remove(rootProperty.Name);
}
else if (rootProperty.Name.Equals("similarity", StringComparison.InvariantCultureIgnoreCase))
{
result.Similarity = serializer.Deserialize<SimilaritySettings>(rootProperty.Value.CreateReader());
result.Settings.Remove(rootProperty.Name);
}
}
}
if (jsonObject["aliases"] != null)
{
var a = serializer.Deserialize<Dictionary<string, CreateAliasOperation>>(jsonObject["aliases"].CreateReader());
result.Aliases = a.ToDictionary(kv => kv.Key, kv => kv.Value as ICreateAliasOperation);
}
if (jsonObject["mappings"] != null)
{
var mappings = serializer.Deserialize<Dictionary<string, RootObjectMapping>>(jsonObject["mappings"].CreateReader());
result.Mappings = mappings.Select(kv =>
{
var name = kv.Key;
kv.Value.Name = name;
return kv.Value;
}).ToList();
}
if (jsonObject["warmers"] != null)
{
var warmers = serializer.Deserialize<Dictionary<string, WarmerMapping>>(jsonObject["warmers"].CreateReader());
result.Warmers = warmers.ToDictionary(kv=>kv.Key, kv =>
{
kv.Value.Name = kv.Key;
return kv.Value;
});
}
return result;
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue,
JsonSerializer serializer)
{
JObject o = JObject.Load(reader);
var result = new IndexSettings();
var dictionary = new Dictionary<string, object>();
serializer.Populate(o.CreateReader(), dictionary);
result.Settings = dictionary;
result.AsExpando = DynamicDictionary.Create(dictionary);
foreach (var rootProperty in o.Children<JProperty>())
{
if (rootProperty.Name.Equals("analysis", StringComparison.InvariantCultureIgnoreCase))
{
result.Analysis = serializer.Deserialize<AnalysisSettings>(rootProperty.Value.CreateReader());
result.Settings.Remove(rootProperty.Name);
}
else if (rootProperty.Name.Equals("warmers", StringComparison.InvariantCultureIgnoreCase))
{
foreach (var jWarmer in rootProperty.Value.Children<JProperty>())
{
result.Warmers[jWarmer.Name] = serializer.Deserialize<WarmerMapping>(jWarmer.Value.CreateReader());
}
result.Settings.Remove(rootProperty.Name);
}
else if (rootProperty.Name.Equals("similarity", StringComparison.InvariantCultureIgnoreCase))
{
result.Similarity = serializer.Deserialize<SimilaritySettings>(rootProperty.Value.CreateReader());
result.Settings.Remove(rootProperty.Name);
}
}
return result;
}
public void UpdateSettingsSimple()
{
var index = Guid.NewGuid().ToString();
var client = this.Client;
var settings = new IndexSettings();
settings.NumberOfReplicas = 1;
settings.NumberOfShards = 5;
settings.Settings.Add("refresh_interval", "1s");
settings.Settings.Add("search.slowlog.threshold.fetch.warn", "1s");
client.CreateIndex(index, i=>i.InitializeUsing(settings));
settings.Settings["refresh_interval"] = "-1";
var r = this.Client.UpdateSettings(us=>us
.Index(index)
.RefreshInterval("-1")
);
Assert.True(r.IsValid);
Assert.True(r.Acknowledged);
var getResponse = this.Client.GetIndexSettings(i=>i.Index(index));
Assert.AreEqual(getResponse.IndexSettings.Settings["refresh_interval"], "-1");
this.Client.DeleteIndex(i=>i.Index(index));
}
public void GetIndexSettingsComplex()
{
var index = Guid.NewGuid().ToString();
var settings = new IndexSettings();
settings.NumberOfReplicas = 4;
settings.NumberOfShards = 8;
settings.Analysis.Analyzers.Add("snowball", new SnowballAnalyzer { Language = "English" });
settings.Analysis.Analyzers.Add("standard", new StandardAnalyzer { StopWords = new[]{"word1", "word2"}});
settings.Analysis.Analyzers.Add("swedishlanguage", new LanguageAnalyzer(Language.Swedish) { StopWords = new[] { "word1", "word2" }, StemExclusionList = new[] { "stem1", "stem2" } });
settings.Analysis.CharFilters.Add("char1", new HtmlStripCharFilter());
settings.Analysis.CharFilters.Add("char2", new MappingCharFilter{ Mappings = new []{"ph=>f", "qu=>q"}});
settings.Analysis.CharFilters.Add("char3", new PatternReplaceCharFilter { Pattern = "sample(.*)", Replacement = "replacedSample $1" });
settings.Analysis.TokenFilters.Add("tokenfilter1", new EdgeNGramTokenFilter());
settings.Analysis.TokenFilters.Add("tokenfilter2", new SnowballTokenFilter());
settings.Analysis.Tokenizers.Add("token1", new KeywordTokenizer());
settings.Analysis.Tokenizers.Add("token2", new PathHierarchyTokenizer());
settings.Similarity.CustomSimilarities.Add("test1", new DFRSimilarity
{
BasicModel = "be",
AfterEffect = "l",
Normalization = "h2",
});
settings.Similarity.CustomSimilarities.Add("test2", new IBSimilarity
{
Distribution = "spl",
Lambda = "ttf",
Normalization = "h1"
});
var typeMappingResult = this.Client.GetMapping<ElasticsearchProject>(gm=>gm.Index(ElasticsearchConfiguration.DefaultIndex).Type("elasticsearchprojects"));
var typeMapping = typeMappingResult.Mapping;
typeMapping.Name = index;
settings.Mappings.Add(typeMapping);
settings.Settings.Add("merge.policy.merge_factor", "10");
var createResponse = this.Client.CreateIndex(index, i=>i.InitializeUsing(settings));
var r = this.Client.GetIndexSettings(i=>i.Index(index));
Assert.True(r.IsValid);
Assert.NotNull(r.IndexSettings.Settings.Count);
Assert.AreEqual(r.IndexSettings.NumberOfReplicas, 4);
Assert.AreEqual(r.IndexSettings.NumberOfShards, 8);
//Assert.Greater(r.Setttings, 0);
Assert.NotNull(r.IndexSettings.AsExpando.merge.policy.merge_factor);
Assert.AreEqual(10, r.IndexSettings.AsExpando.merge.policy.merge_factor);
Assert.AreEqual(3, r.IndexSettings.Analysis.Analyzers.Count);
{ // assert analyzers
Assert.True(r.IndexSettings.Analysis.Analyzers.ContainsKey("snowball"));
var snoballAnalyser = r.IndexSettings.Analysis.Analyzers["snowball"] as SnowballAnalyzer;
Assert.NotNull(snoballAnalyser);
Assert.AreEqual("English", snoballAnalyser.Language);
Assert.True(r.IndexSettings.Analysis.Analyzers.ContainsKey("standard"));
var standardAnalyser = r.IndexSettings.Analysis.Analyzers["standard"] as StandardAnalyzer;
Assert.NotNull(standardAnalyser);
Assert.NotNull(standardAnalyser.StopWords);
Assert.AreEqual(2, standardAnalyser.StopWords.Count());
Assert.True(standardAnalyser.StopWords.Contains("word1"));
Assert.True(standardAnalyser.StopWords.Contains("word2"));
Assert.True(r.IndexSettings.Analysis.Analyzers.ContainsKey("swedishlanguage"));
var languageAnalyser = r.IndexSettings.Analysis.Analyzers["swedishlanguage"] as LanguageAnalyzer;
Assert.NotNull(languageAnalyser);
Assert.AreEqual(Language.Swedish.ToString().ToLowerInvariant(), languageAnalyser.Type);
Assert.NotNull(languageAnalyser.StopWords);
Assert.AreEqual(2, languageAnalyser.StopWords.Count());
Assert.True(languageAnalyser.StopWords.Contains("word1"));
Assert.True(languageAnalyser.StopWords.Contains("word2"));
Assert.AreEqual(2, languageAnalyser.StemExclusionList.Count());
Assert.True(languageAnalyser.StemExclusionList.Contains("stem1"));
Assert.True(languageAnalyser.StemExclusionList.Contains("stem2"));
}
Assert.AreEqual(3, r.IndexSettings.Analysis.CharFilters.Count);
{ // assert char filters
Assert.True(r.IndexSettings.Analysis.CharFilters.ContainsKey("char1"));
var filter1 = r.IndexSettings.Analysis.CharFilters["char1"] as HtmlStripCharFilter;
Assert.NotNull(filter1);
Assert.True(r.IndexSettings.Analysis.CharFilters.ContainsKey("char2"));
var filter2 = r.IndexSettings.Analysis.CharFilters["char2"] as MappingCharFilter;
Assert.NotNull(filter2);
Assert.AreEqual(2, filter2.Mappings.Count());
Assert.True(filter2.Mappings.Contains("ph=>f"));
Assert.True(filter2.Mappings.Contains("qu=>q"));
var filter3 = r.IndexSettings.Analysis.CharFilters["char3"] as PatternReplaceCharFilter;
Assert.NotNull(filter3);
Assert.AreEqual("sample(.*)", filter3.Pattern);
Assert.AreEqual("replacedSample $1", filter3.Replacement);
}
Assert.AreEqual(2, r.IndexSettings.Analysis.TokenFilters.Count);
{ // assert token filters
Assert.True(r.IndexSettings.Analysis.TokenFilters.ContainsKey("tokenfilter1"));
var filter1 = r.IndexSettings.Analysis.TokenFilters["tokenfilter1"] as EdgeNGramTokenFilter;
Assert.NotNull(filter1);
Assert.True(r.IndexSettings.Analysis.TokenFilters.ContainsKey("tokenfilter2"));
var filter2 = r.IndexSettings.Analysis.TokenFilters["tokenfilter2"] as SnowballTokenFilter;
Assert.NotNull(filter2);
}
Assert.AreEqual(2, r.IndexSettings.Analysis.Tokenizers.Count);
{ // assert tokenizers
Assert.True(r.IndexSettings.Analysis.Tokenizers.ContainsKey("token1"));
var tokenizer1 = r.IndexSettings.Analysis.Tokenizers["token1"] as KeywordTokenizer;
Assert.NotNull(tokenizer1);
Assert.True(r.IndexSettings.Analysis.Tokenizers.ContainsKey("token2"));
var tokenizer2 = r.IndexSettings.Analysis.Tokenizers["token2"] as PathHierarchyTokenizer;
Assert.NotNull(tokenizer2);
}
Assert.NotNull(r.IndexSettings.Similarity);
Assert.NotNull(r.IndexSettings.Similarity.CustomSimilarities);
Assert.AreEqual(2, r.IndexSettings.Similarity.CustomSimilarities.Count);
{ // assert similarity
var similarity1 = r.IndexSettings.Similarity.CustomSimilarities.FirstOrDefault(x => x.Key.Equals("test1", StringComparison.InvariantCultureIgnoreCase)).Value as DFRSimilarity;
Assert.NotNull(similarity1);
Assert.AreEqual("DFR", similarity1.Type);
Assert.AreEqual("be", similarity1.BasicModel);
Assert.AreEqual("l", similarity1.AfterEffect);
Assert.AreEqual("h2", similarity1.Normalization);
var similarity2 = r.IndexSettings.Similarity.CustomSimilarities.FirstOrDefault(x => x.Key.Equals("test2", StringComparison.InvariantCultureIgnoreCase)).Value as IBSimilarity;
Assert.NotNull(similarity2);
Assert.AreEqual("IB", similarity2.Type);
Assert.AreEqual("spl", similarity2.Distribution);
Assert.AreEqual("ttf", similarity2.Lambda);
Assert.AreEqual("h1", similarity2.Normalization);
}
this.Client.DeleteIndex(i=>i.Index(index));
}
public LarmSettings()
{
Aws = new AwsSettings();
Index = new IndexSettings();
}
public void UpdateSettingsSimple()
{
var index = Guid.NewGuid().ToString();
var client = this._client;
var settings = new IndexSettings();
settings.NumberOfReplicas = 1;
settings.NumberOfShards = 5;
settings.Add("refresh_interval", "1s");
settings.Add("search.slowlog.threshold.fetch.warn", "1s");
client.CreateIndex(index, settings);
settings["refresh_interval"] = "-1";
settings["search.slowlog.threshold.fetch.warn"] = "5s";
var r = this._client.UpdateSettings(index, settings);
Assert.True(r.IsValid);
Assert.True(r.OK);
var getResponse = this._client.GetIndexSettings(index);
Assert.AreEqual(getResponse.Settings["refresh_interval"], "-1");
Assert.AreEqual(getResponse.Settings["search.slowlog.threshold.fetch.warn"], "1s");
this._client.DeleteIndex(index);
}
private void buildAndSaveIndex(MapAround.Mapping.FeatureType featureType,
BoundingRectangle b,
IndexSettings settings,
IEnumerable<Feature> features)
{
ISpatialIndex index = null;
if (b.IsEmpty())
b = new BoundingRectangle(0, 0, 0, 0);
if (settings.IndexType == "QuadTree")
index = new QuadTree(b);
if (index == null)
index = new KDTree(b);
index.MaxDepth = settings.MaxDepth;
index.BoxSquareThreshold = settings.BoxSquareThreshold;
index.MinObjectCount = settings.MinFeatureCount;
index.Build(features);
_cacheAccessor.SaveFeaturesIndex(index, featureType);
}
