public void CreateIndex()
{
if (indexQuery.IsExists(Constants.SlambyMetadataIndex))
{
return;
}
var client = elasticClientFactory.GetClient();
var descriptor = new CreateIndexDescriptor(Constants.SlambyMetadataIndex);
descriptor
.Settings(s => s
.NumberOfReplicas(0)
.NumberOfShards(1))
.Mappings(m => m
.Map <MetadataElastic>(mm => mm
.AutoMap()
.Dynamic(false)
));
var createResp = client.CreateIndex(descriptor);
ResponseValidator(createResp);
var metadataElastic = new MetadataElastic()
{
DBVersion = 0
};
Index(metadataElastic);
}
private void MapModelElastic <TModelElastic>(string index) where TModelElastic : class
{
var elasticClient = clientFactory.GetClient(index);
var mapresponse = elasticClient.Map <TModelElastic>(m => m
.AutoMap()
.Dynamic(DynamicMapping.Strict));
if (!mapresponse.Acknowledged)
{
throw new Exception("ElasticSearch not acknowledged the Update!");
}
}
private void ConfigureDependencies(IServiceCollection services)
{
//see: https://github.com/aspnet/Hosting/issues/793
services.AddSingleton <IHttpContextAccessor, HttpContextAccessor>();
services.AddSingleton(sp => sp.GetService <IOptions <SiteConfig> >().Value);
services.AddTransient(sp => ElasticClientFactory.GetClient(sp));
// Add DependencyAttribute (ScopedDependency, SingletonDependency, TransientDependency) to the class in order to be scanned
services.ConfigureAttributedDependencies();
{
var options = ConfigurationOptions.Parse(Configuration["SlambyApi:Redis:Configuration"]);
//HACK: https://github.com/dotnet/corefx/issues/8768
//this should be removed when https://github.com/dotnet/corefx/issues/11564 is closed
options = RedisDnsHelper.CorrectOption(options);
if (options == null)
{
Log.Logger.Error("Can't resolve the name of the Redis server!");
return;
}
services.AddSingleton(RedisDnsHelper.CorrectOption(options));
services.AddSingleton <ConnectionMultiplexer>(sp => ConnectionMultiplexer.Connect(sp.GetService <ConfigurationOptions>()));
}
}
public void CreateIndex()
{
if (indexQuery.IsExists(Constants.SlambyProcessesIndex))
{
return;
}
var client = elasticClientFactory.GetClient();
var descriptor = new CreateIndexDescriptor(Constants.SlambyProcessesIndex);
descriptor
.Settings(s => s.
NumberOfReplicas(0)
.NumberOfShards(1))
.Mappings(m => m
.Map <ProcessElastic>(mm => mm.AutoMap().Dynamic(false))
);
var createResp = client.CreateIndex(descriptor);
ResponseValidator(createResp);
var propDesc = new PropertiesDescriptor <ProcessElastic>();
propDesc.Object <object>(s => s.Name(ProcessElastic.InitObjectMappingName).Dynamic(true));
var putMappingDesc = new PutMappingDescriptor <ProcessElastic>()
.Index(Constants.SlambyProcessesIndex)
.Dynamic(DynamicMapping.Strict)
.Properties(p => propDesc);
var mapResp = client.Map <DocumentElastic>(p => putMappingDesc);
ResponseValidator(mapResp);
}
public void CreateIndex()
{
if (indexQuery.IsExists(Constants.SlambyServicesIndex))
{
return;
}
var descriptor = new CreateIndexDescriptor(Constants.SlambyServicesIndex);
descriptor
.Settings(s => s
.NumberOfReplicas(0)
.NumberOfShards(1))
.Mappings(m => m
.Map <ServiceElastic>(mm => mm.AutoMap().Dynamic(false))
.Map <ClassifierSettingsElastic>(mm => mm.AutoMap().Dynamic(false))
.Map <PrcSettingsElastic>(mm => mm.AutoMap().Dynamic(false)));
var createResp = elasticClientFactory.GetClient().CreateIndex(descriptor);
ResponseValidator(createResp);
}
public void CreateIndex()
{
if (indexQuery.IsExists(Constants.SlambyFileParserIndex))
{
return;
}
var descriptor = new CreateIndexDescriptor(Constants.SlambyFileParserIndex);
descriptor
.Settings(s => s
.NumberOfReplicas(0)
.NumberOfShards(1))
.Mappings(m => m
.Map <FileParserElastic>(mm => mm
.Properties(
p => p.Attachment(desc => desc
.Name("content")
.FileField(d => d //ContentField
.Store(true)
.Analyzer("standard"))
.ContentTypeField(d => d.Store(true))
.ContentLengthField(d => (d as NumberPropertyDescriptor <FileParserElastic>).Store(true))
.LanguageField(d => (d as StringPropertyDescriptor <FileParserElastic>).Store(true))
.KeywordsField(d => d.Store(true))
.AuthorField(d => d.Store(true))
.DateField(d => d.Store(true))
.TitleField(d => d.Store(true))
)
))
);
var createResp = elasticClientFactory.GetClient().CreateIndex(descriptor);
ResponseValidator(createResp);
}
private void WaitForElastic()
{
var available = false;
var counter = 1;
do
{
var isValid = false;
var status = string.Empty;
try
{
var client = elasticClientFactory.GetClient();
var health = client.ClusterHealth();
isValid = health.IsValid;
status = health.Status;
}
catch
{
isValid = false;
}
if (isValid)
{
if (status != null && !string.Equals(status, "red", StringComparison.OrdinalIgnoreCase))
{
available = true;
logger.LogInformation($"Elasticsearch connection is Available ({status})");
break;
}
logger.LogInformation($"Elasticsearch connection is Online ({counter}) ({status})");
}
else
{
logger.LogWarning($"Elasticsearch connection is Offline ({counter})");
}
System.Threading.Thread.Sleep(2000);
}while ((counter++) < 300);
if (!available)
{
throw new TimeoutException("Elasticsearch is Offline! Check ES settings!");
}
}
public async Task <CheckResponseModel> RequestCheck(string xmlText, Guid instanceId, DateTime launchTime)
{
try
{
var elasticName = clientFactory.GetClient().RootNodeInfo()?.Name ?? string.Empty;
using (var client = CreateHttpClient())
{
var model = new CheckRequestModel()
{
Id = instanceId,
License = xmlText.ToBase64(),
Details = new CheckDetailsModel()
{
LaunchTime = launchTime,
Cores = Environment.ProcessorCount,
ElasticName = elasticName
}
};
var json = JsonConvert.SerializeObject(model);
var content = new StringContent(json, Encoding.UTF8, "application/json");
var response = await client.PostAsync("api/check", content);
if (response.StatusCode != System.Net.HttpStatusCode.BadRequest &&
response.StatusCode != System.Net.HttpStatusCode.OK)
{
return(null);
}
var responseBody = await response.Content.ReadAsStringAsync();
var checkResponse = JsonConvert.DeserializeObject <CheckResponseModel>(responseBody);
return(checkResponse);
}
}
catch (Exception)
{
return(null);
}
}
public void CreateIndex(string name, string indexName, int shingleCount, object sampleDynamicDocument, string idField, List <string> interpretedFields, string tagField)
{
try
{
#region Default Index creation
var tokenFilters = CreateTokenFilterDescriptor();
var analyzers = CreateAnalyzersDescriptor(shingleCount, DefaultTokenizer, DefaultCharFilter, tokenFilters);
var analysisDescriptor = CreateAnalysisDescriptor(DefaultCharFilter, tokenFilters, analyzers);
var indexDescriptor = CreateIndexDescriptor(indexName, analysisDescriptor,
(map) => map
.Dynamic(true)
.DateDetection(false)
.NumericDetection(false)
.AutoMap());
var createResp = defaultClient.CreateIndex(indexDescriptor);
ResponseValidator(createResp);
#endregion
var client = clientFactory.GetClient(indexName);
#region Dynamic Document indexing
var dynamicDocumentType = DocumentElastic.DocumentTypeName;
var sampleId = "sample_id";
var req = new IndexRequest <DocumentElastic>(new DocumentElastic {
DocumentObject = sampleDynamicDocument
}, indexName, dynamicDocumentType, sampleId);
var indexResp = client.Index <object>(req);
ResponseValidator(indexResp);
var deleteResp = client.Delete(new DeleteRequest(indexName, dynamicDocumentType, sampleId));
ResponseValidator(deleteResp);
#endregion
client.Flush(indexName);
#region Malformed string for Date and Number types
//if a number or date property is a string in the json, then it makes an exception during the index if we don't put IgnoreMalformed on the property
var mappingResponse = client.GetMapping(new GetMappingRequest(indexName, dynamicDocumentType));
ResponseValidator(mappingResponse);
var prop = mappingResponse.Mapping.Properties[DocumentElastic.DocumentObjectMappingName];
var canMalformedPropsKvp = _setIgnoreMalformedDateAndNumberPropertyNames(prop);
var canMalformedProps = new Properties(canMalformedPropsKvp.GroupBy(x => x.Key).Select(g => g.First()).ToDictionary(c => c.Key, c => c.Value));
var canMalformedPropDesc = new PropertiesDescriptor <object>(canMalformedProps);
var putMappingDescMalformed = new PutMappingDescriptor <DocumentElastic>()
.Index(indexName)
.Properties(p => p.Object <object>(o => o.Name(DocumentElastic.DocumentObjectMappingName).Properties(op => canMalformedPropDesc)));
var mapMalformedResp = client.Map <DocumentElastic>(p => putMappingDescMalformed);
ResponseValidator(mapMalformedResp);
#endregion
#region Interpreted fields and text field mapping
var ipfPropDesc = CreateInterpretedFieldsPropertyDescriptor();
var interpretedFieldsPropDesc = new PropertiesDescriptor <object>();
var segments = interpretedFields.OrderBy(field => field)
.Select(field => field
.Split(new[] { "." }, StringSplitOptions.None))
.ToArray();
IndexHelper.SetInterpretedFields(interpretedFieldsPropDesc, ipfPropDesc, segments);
var multiPropDesc = CreateAnalyzerPropertiesDescriptor(shingleCount);
var putMappingDesc = new PutMappingDescriptor <DocumentElastic>()
.Index(indexName)
.Dynamic(DynamicMapping.Strict)
.Properties(pr => pr
.String(map => map
.Name(DocumentElastic.TextField)
.Fields(f => multiPropDesc))
.Object <object>(map => map
.Name(DocumentElastic.DocumentObjectMappingName)
.Properties(op => interpretedFieldsPropDesc)));
var mapResp = client.Map <DocumentElastic>(p => putMappingDesc);
ResponseValidator(mapResp);
#endregion
client.Flush(indexName);
#region Add index property
AddIndexProperty(client, name, shingleCount, sampleDynamicDocument, null, idField, interpretedFields, tagField);
#endregion
client.Flush(indexName);
CreateAlias(indexName, name);
}
catch (Exception ex)
{
defaultClient.DeleteIndex(indexName);
throw ex;
}
}
public IndexQuery(ElasticClient client, ElasticClientFactory clientFactory, SiteConfig siteConfig) : base(client, siteConfig)
{
this.clientFactory = clientFactory;
this.defaultClient = clientFactory.GetClient();
}
public BaseQuery(ElasticClientFactory elasticClientFactory, string indexName, SiteConfig siteConfig)
{
IndexName = indexName;
Client = elasticClientFactory.GetClient(indexName);
SiteConfig = siteConfig;
}
static void Main(string[] args)
{
Randomizer.Seed = new Random(3897234);
var random = new Bogus.Randomizer();
var guildNames = new[] { null, "Guild A", "Guild B", "Guild C", "Guild D", "Guild E" };
var faction = new [] { "Horde", "Alliance", "Undefined" };
var race = new[]
{
"Human", "Kaldorei", "Gnome", "Dwarf", "Worgen", "Orc", "Tauren",
"Forsaken", "Sindorei", "Goblin", "Pandaren"
};
var isActive = new[] { true, false };
var dateInfo = new Faker <DateInfo>()
.StrictMode(true)
.RuleFor(x => x.CreatedDate, f => f.Date.Recent())
.RuleFor(x => x.UpdatedDate, f => f.Date.Recent());
var story = new Faker <Story>()
.StrictMode(true)
.RuleFor(x => x.Body, f => f.Lorem.Paragraphs(3))
.RuleFor(x => x.Id, f => f.IndexFaker)
.RuleFor(x => x.Title, f => f.Lorem.Sentence())
.RuleFor(x => x.DateInfo, f => dateInfo.Generate());
var picture = new Faker <Picture>()
.StrictMode(true)
.RuleFor(x => x.Id, f => f.IndexFaker)
.RuleFor(x => x.Caption, f => f.Lorem.Sentence())
.RuleFor(x => x.Url, f => f.Internet.Url());
var guild = new Faker <Guild>()
.StrictMode(true)
.RuleFor(x => x.Name, f => f.PickRandom(guildNames))
.RuleFor(x => x.Description, f => f.Lorem.Paragraphs(3));
var character = new Faker <Character>()
.StrictMode(true)
.RuleFor(x => x.Id, f => f.IndexFaker)
.RuleFor(x => x.FirstName, f => f.Name.FirstName())
.RuleFor(x => x.LastName, f => f.Name.LastName())
.RuleFor(x => x.EyeColor, f => f.Commerce.Color())
.RuleFor(x => x.EyeColor, f => f.Commerce.Color())
.RuleFor(x => x.Weight, f => f.Lorem.Sentence())
.RuleFor(x => x.Height, f => f.Lorem.Sentence())
.RuleFor(x => x.SkinColor, f => f.Commerce.Color())
.RuleFor(x => x.Race, f => f.PickRandom(race))
.RuleFor(x => x.Faction, f => f.PickRandom(faction))
.RuleFor(x => x.Background, f => f.Lorem.Paragraphs(7))
.RuleFor(x => x.Residence, f => f.Lorem.Word())
.RuleFor(x => x.Occupation, f => f.Lorem.Sentence())
.RuleFor(x => x.PhysicalAppearance, f => f.Lorem.Paragraphs(2))
.RuleFor(x => x.Friends, f => f.Lorem.Words().ToList())
.RuleFor(x => x.Stories, f => story.Generate(random.Number(1, 10)).ToList())
.RuleFor(x => x.Pictures, f => picture.Generate(random.Number(1, 10)).ToList())
.RuleFor(x => x.Guild, f => guild.Generate())
.RuleFor(x => x.DateInfo, f => dateInfo.Generate());
var user = new Faker <User>()
.StrictMode(true)
.RuleFor(x => x.UserName, f => f.Internet.UserName())
.RuleFor(x => x.DateInfo, dateInfo.Generate())
.RuleFor(x => x.Email, f => f.Internet.Email())
.RuleFor(x => x.Image, f => f.Internet.Url())
.RuleFor(x => x.IsActive, f => f.PickRandom(isActive))
.RuleFor(x => x.Characters, f => character.Generate(random.Number(1, 10)).ToList());
var users = user.Generate(1000);
//string charsJson = JsonConvert.SerializeObject(users);
ElasticClient client = ElasticClientFactory.GetClient();
//File.WriteAllText(@"c:\dump\users.txt", charsJson);
var result = client.Bulk(b => b.IndexMany(users));
if (!result.IsValid)
{
foreach (var item in result.ItemsWithErrors)
{
Console.WriteLine("Failed to index document {0}: {1}", item.Id, item.Error);
}
Console.WriteLine(result.DebugInformation);
Console.Read();
Environment.Exit(1);
}
Console.WriteLine("Done!");
Console.ReadLine();
}
