public MemberElasticSearchIndex(string name, ElasticSearchConfig connectionConfiguration,
IProfilingLogger profilingLogger,
FieldDefinitionCollection fieldDefinitions = null, string analyzer = null,
IValueSetValidator validator = null) : base(name, connectionConfiguration,
profilingLogger, fieldDefinitions, analyzer, validator)
{
}
public TestBaseIndex(ElasticSearchConfig connectionConfiguration,
FieldDefinitionCollection fieldDefinitions = null,
string analyzer = null,
IValueSetValidator validator = null)
: base("testIndexer", connectionConfiguration, fieldDefinitions, analyzer, validator)
{
}
/// <summary>
/// 获取配置文件
/// </summary>
private void GetJsonConfig()
{
AppsettingsConfig config = new AppsettingsConfig();
AppsettingsConfig.ServiceApiHosts = new ServiceApiHosts();
Configuration.Bind("WebConfig", config);
ElasticSearchConfig elasticSearchConfig = new ElasticSearchConfig();
ElasticSearchConfig.ClusterNodeUrlHosts = new string[0];
Configuration.Bind("ElasticSearchConfig", elasticSearchConfig);
var aipHostSection = Configuration.GetSection("ServiceApiHosts");
var dic = new Dictionary <string, string>();
if (aipHostSection != null)
{
foreach (var api in aipHostSection.GetChildren())
{
dic.Add(api["Code"], api["Host"]);
}
}
AppsettingsConfig.ServiceApiHosts = new ServiceApiHosts();
AppsettingsConfig.ServiceApiHosts.ApiHostOptions = new Dictionary <string, string>();
AppsettingsConfig.ServiceApiHosts.ApiHostOptions = dic;
}
public DomainEventStore(ElasticSearchConfig config, IMessageSerializer messageSerializer)
{
this.config = config;
this.messageSerializer = messageSerializer;
this.elasticClient = CreateElasticClient();
CheckIndexExists();
}
public AggregateSnapshotAccessor(SchemaVersion version, ElasticSearchConfig config)
{
this.version = version;
this.config = config;
this.elasticClient = this.CreateElasticClient();
this.CheckIndexExists();
}
public void Can_Delete()
{
using (var elasticsearch = new ElasticsearchInside.Elasticsearch(settings => settings
.EnableLogging()
.SetPort(9200)
.SetElasticsearchStartTimeout(180)).ReadySync())
{
ElasticSearchConfig config = new ElasticSearchConfig(new ConnectionSettings(elasticsearch.Url));
using (var indexer = new TestBaseIndex(config, new FieldDefinitionCollection()))
{
for (var i = 0; i < 10; i++)
{
indexer.IndexItem(new ValueSet(i.ToString(), "content",
new Dictionary <string, IEnumerable <object> >
{
{ "item1", new List <object>(new[] { "value1" }) },
{ "item2", new List <object>(new[] { "value2" }) }
}));
}
indexer.DeleteFromIndex("9");
indexer._client.Value.Indices.Refresh(Indices.Index(indexer.indexAlias));
Assert.AreEqual(9, indexer.DocumentCount);
}
}
}
public ContentElasticSearchIndex(string name, ElasticSearchConfig connectionConfiguration, IProfilingLogger profilingLogger,
FieldDefinitionCollection fieldDefinitions = null, string analyzer = null,
IValueSetValidator validator = null, bool publishedValuesOnly = false) : base(name, connectionConfiguration,
profilingLogger, fieldDefinitions, analyzer, validator)
{
PublishedValuesOnly = publishedValuesOnly;
}
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
services.AddSingleton <ElasticSearchConfig>(x =>
{
var esConfig = new ElasticSearchConfig();
this.Configuration.Bind(esConfig);
return(esConfig);
});
services.AddSingleton <KafkaConfig>(x => new KafkaConfig
{
KAFKA_CONSUMER_CONFIG = JsonConvert
.DeserializeObject <Dictionary <string, object> >(
this.Configuration.GetValue <string>(
nameof(KafkaConfig.KAFKA_CONSUMER_CONFIG))),
KAFKA_TOPICS = JsonConvert
.DeserializeObject <string[]>(
this.Configuration.GetValue <string>(
nameof(KafkaConfig.KAFKA_TOPICS))),
});
services.AddSingleton <IElasticSearchAccessor, ElasticSearchAccessor>();
services.AddSingleton <IMessageHandler, MessageHandler>();
services.AddSingleton <IMessagingQueue, MessagingQueue>();
services.AddMvc();
}
public GameSearchService(IOptions <ElasticSearchConfig> config, ElasticService client, IWorkerQueue workerQueue, ILogger <GameSearchService> logger)
{
_config = config.Value;
_client = client;
_workerQueue = workerQueue;
_logger = logger;
}
public static ConnectionSettings Create(ElasticSearchConfig elasticConfig)
{
ConnectionSettings settings = null;
if (!string.IsNullOrEmpty(elasticConfig.Url))
{
settings = new ConnectionSettings(new Uri(elasticConfig.Url));
}
else
{
settings = new ConnectionSettings(
elasticConfig.CloudId,
new Elasticsearch.Net.BasicAuthenticationCredentials(elasticConfig.Username, elasticConfig.Password));
}
settings.DisableDirectStreaming(true);
settings.OnRequestCompleted(call =>
{
if (call.RequestBodyInBytes != null)
{
var request = Encoding.UTF8.GetString(call.RequestBodyInBytes);
Console.WriteLine(request);
}
});
return(settings);
}
/// <summary>
/// 注册Elasticsearch日志操作
/// </summary>
/// <param name="services">服务集合</param>
/// <param name="setupAction">配置操作</param>
public static void AddElasticsearch(this IServiceCollection services, Action <ElasticSearchConfig> setupAction)
{
var config = new ElasticSearchConfig();
setupAction?.Invoke(config);
services.TryAddSingleton <IElasticSearchConfigProvider>(new DefaultElasticSearchConfigProvider(config));
services.TryAddScoped <IElasticSearchManager, ElasticSearchManager>();
}
public virtual IEnumerable <IIndex> Create() => (IEnumerable <IIndex>) new PdfElasticIndex[1]
{
new PdfElasticIndex("PDFIndex", ElasticSearchConfig.GetConfig("PDFIndex"), new FieldDefinitionCollection(
new FieldDefinition[1]
{
new FieldDefinition("fileTextContent", "fulltext")
}), "standard", new PdfValueSetValidator(new int?()), false)
};
private IIndex CreateMemberIndex()
{
return(new MemberElasticSearchIndex(Constants.UmbracoIndexes.MembersIndexName,
ElasticSearchConfig.GetConfig(Constants.UmbracoIndexes.ExternalIndexName),
ProfilingLogger,
new UmbracoFieldDefinitionCollection(),
"standard",
UmbracoIndexConfig.GetMemberValueSetValidator()));
}
private IIndex CreateExternalIndex()
{
return(new ContentElasticSearchIndex(Constants.UmbracoIndexes.ExternalIndexName,
ElasticSearchConfig.GetConfig(Constants.UmbracoIndexes.ExternalIndexName),
ProfilingLogger,
new UmbracoFieldDefinitionCollection(),
"standard",
UmbracoIndexConfig.GetPublishedContentValueSetValidator(), true));
}
private IIndex CreateInternalIndex()
{
return(new ContentElasticSearchIndex(Constants.UmbracoIndexes.InternalIndexName,
ElasticSearchConfig.GetConfig(Constants.UmbracoIndexes.InternalIndexName),
ProfilingLogger,
new UmbracoFieldDefinitionCollection(),
"whitespace",
UmbracoIndexConfig.GetContentValueSetValidator()));
}
public SearchResultPTO FullTextSearch(string query, int page = 1, int pageSize = 10)
{
int offset = 0;
if (page > 1)
{
offset = (page - 1) * pageSize;
}
// client to interact with elasticSearch
client = ElasticSearchConfig.GetClient();
// query to run against the Author, Title and name field
var result = client.Search <Document>(x => x
.Query(q => q
.MultiMatch(mp => mp
.Query(query)
.Fields(f => f
.Fields(f1 => f1.Title, f2 => f2.Subtitle))))
.Highlight(h => h
.PreTags("<bold style=\"color: blue; \">")
.PostTags("</bold>")
.Fields(fs => fs
.Field(p => p.Title) // Highlight strings in title and Subtitle fields
.Type("plain"), // that match the query
fs => fs
.Field(p => p.Subtitle)
.Type("plain")))
.From(offset) //pagination options
.Size(pageSize));
// return a result object containing resulting documents from Query along with some more
// information about the query results
return(new SearchResultPTO
{
Total = (int)result.Total,
Page = page,
Results = result.Hits.Select(h => new ResultPTO()
{
DocuID = h.Source.DocuID,
DocumentID = h.Source.DocumentID,
FileSize = h.Source.FileSize,
FilePath = "",
Title = (h.Highlights.Values.Count > 1) ? h.Highlights.Values.ElementAt(1).Highlights.First() : h.Source.Title,
Subtitle = (h.Highlights.Values.Count > 0) ? h.Highlights.Values.First().Highlights.First() : h.Source.Subtitle,
FunctionalArea = h.Source.FunctionalArea,
LastModified = h.Source.LastModified,
Release = h.Source.Release,
SubType = h.Source.SubType,
InPackage = false,
PrevInPackage = false
}),
QueryTime = result.Took,
// add the aggregations later on
});
}
public void Create(HttpPostedFileBase upload)
{
// client to interact with elasticSearch
client = ElasticSearchConfig.GetClient();
//Most of Avaloq's document filenames follow the format of ID-ReleaseNum-ClientNum-Subtype-Subtitle.pdf
//The service checks if the given file follows the same format, and if so, retrieves the metadata.
using (var dc = new DocuContext())
{
var document = new Models.Document {
};
var fileName = upload.FileName;
String[] metadata = fileName.Split('-');
if (upload != null && upload.ContentLength > 0 && metadata.Length >= 5)
{
document.SubType = metadata[3];
document.DocuID = Convert.ToInt32(metadata[0]);
if (metadata[1].Equals("en"))
{
document.Release = "Release Independent";
}
else
{
document.Release = "Release " + metadata[1];
}
document.LastModified = DateTime.Today.Date;
document.FileSize = upload.ContentLength;
document.Title = upload.FileName;
document.Subtitle = metadata[4];
var filepath = new Models.FilePath
{
FileName = Path.GetFileName(upload.FileName),
};
document.FilePath = filepath;
}
else if (upload != null && upload.ContentLength > 0)
{
document.SubType = "null";
document.DocuID = 0;
document.Release = "null";
document.LastModified = DateTime.Today.Date;
document.FileSize = upload.ContentLength;
document.Title = upload.FileName;
document.Subtitle = "null";
var filepath = new Models.FilePath
{
FileName = Path.GetFileName(upload.FileName),
};
document.FilePath = filepath;
}
dc.Documents.Add(document);
dc.SaveChanges();
document.FilePath = null;
var result = client.Index(document);
}
}
/// <summary>
/// Create a new <see cref="UmbracoExamineIndex"/>
/// </summary>
/// <param name="name"></param>
/// <param name="fieldDefinitions"></param>
/// <param name="luceneDirectory"></param>
/// <param name="defaultAnalyzer"></param>
/// <param name="profilingLogger"></param>
/// <param name="validator"></param>
/// <param name="indexValueTypes"></param>
protected ElasticSearchUmbracoIndex(string name,
ElasticSearchConfig connectionConfiguration,
IProfilingLogger profilingLogger,
FieldDefinitionCollection fieldDefinitions = null,
string analyzer = null,
IValueSetValidator validator = null)
: base(name, connectionConfiguration, fieldDefinitions, analyzer, validator, true)
{
_logger = profilingLogger;
}
public void ConfigureServices(IServiceCollection services)
{
services.AddOptions();
services.Configure <ScreenshotsConfig>(this.Configuration);
services.AddSingleton <ElasticSearchConfig>(x =>
{
var esConfig = new ElasticSearchConfig();
this.Configuration.Bind(esConfig);
return(esConfig);
});
services.AddSingleton <IDependencyResolver>(s => new FuncDependencyResolver(s.GetRequiredService));
services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_1);
services.AddGraphQL(options =>
{
options.EnableMetrics = true;
options.ExposeExceptions = true;
})
.AddUserContextBuilder(httpContext =>
{
if (httpContext.User.Identity.IsAuthenticated ||
!string.IsNullOrEmpty(this.Configuration.GetValue <string>(nameof(AuthConfig.NoAuth))))
{
return(new GraphQLUserContext {
User = httpContext.User
});
}
if (string.IsNullOrEmpty(this.Configuration.GetValue <string>(nameof(AuthConfig.NoErrors))))
{
throw new UnauthorizedAccessException("Failed to authenticate the client");
}
return(null);
});
services.AddAuthentication(IdentityServerAuthenticationDefaults.AuthenticationScheme)
.AddIdentityServerAuthentication(options =>
{
options.RequireHttpsMetadata = true;
// base-address of your identityserver
options.Authority = this.Configuration
.GetValue <string>("IDENTITY_URL");
// name of the API resource
options.ApiName = "schemes-querier";
options.ApiSecret = this.Configuration.GetValue <string>("IDENTITY_SCHEMES_QUERIER_API_SECRET");
options.EnableCaching = false;
options.CacheDuration = TimeSpan.FromMinutes(4); // default = 10
options.JwtValidationClockSkew = TimeSpan.FromMinutes(4);
});
}
/// <summary>
/// 创建ElasticSearch客户端
/// </summary>
/// <param name="config">配置</param>
/// <returns></returns>
private IElasticClient CreateClient(ElasticSearchConfig config)
{
var nodes = config.ConnectionString.Split('|').Select(x => new Uri(x)).ToList();
var connectionPool = new StaticConnectionPool(nodes);
var connectionSettings = new ConnectionSettings(connectionPool);
connectionSettings.DisablePing(false);
connectionSettings.DisableDirectStreaming(true);
connectionSettings.ThrowExceptions(true);
//connectionSettings.BasicAuthentication(config.UserName, config.Password);
return(new ElasticClient(connectionSettings));
}
private ESNodeManager()
{
_config = ElasticSearchConfig.Instance;
foreach (var esNode in _config.Clusters)
{
BuildCluster(esNode.ClusterName,esNode.TransportType);
ClusterThriftNodes[esNode.ClusterName] = BuildThriftNodes(esNode.ThriftNodes);
ClusterHttpNodes[esNode.ClusterName] = BuildHttpNodes(esNode.HttpNodes);
}
_rand = new Random((int) DateTime.Now.Ticks);
ElasticSearchConfig.ConfigChanged += ElasticSearchConfig_ConfigChanged;
}
private ESNodeManager()
{
_config = ElasticSearchConfig.Instance;
foreach (var esNode in _config.Clusters)
{
BuildCluster(esNode.ClusterName, esNode.TransportType);
ClusterThriftNodes[esNode.ClusterName] = BuildThriftNodes(esNode.ThriftNodes);
ClusterHttpNodes[esNode.ClusterName] = BuildHttpNodes(esNode.HttpNodes);
}
_rand = new Random((int)DateTime.Now.Ticks);
ElasticSearchConfig.ConfigChanged += ElasticSearchConfig_ConfigChanged;
}
// This method gets called by the runtime. Use this method to add services to the container.
public virtual void /*IServiceProvider*/ ConfigureServices(IServiceCollection services)
{
services.AddOptions();
services.AddMediatR();
services.Configure <AggregatesConfig>(Configuration);
services.AddSingleton <ElasticSearchConfig>(x =>
{
var esConfig = new ElasticSearchConfig();
Configuration.Bind(esConfig);
return(esConfig);
});
services.AddSingleton <KafkaConfig>(x => new KafkaConfig
{
KAFKA_EVENTS_CONSUMER_CONFIG = JsonConvert
.DeserializeObject <Dictionary <string, object> >(
Configuration.GetValue <string>(
nameof(KafkaConfig.KAFKA_EVENTS_CONSUMER_CONFIG))),
KAFKA_REQUESTS_CONSUMER_CONFIG = JsonConvert
.DeserializeObject <Dictionary <string, object> >(
Configuration.GetValue <string>(
nameof(KafkaConfig.KAFKA_REQUESTS_CONSUMER_CONFIG))),
KAFKA_EVENTS_PRODUCER_CONFIG = JsonConvert
.DeserializeObject <Dictionary <string, object> >(
Configuration.GetValue <string>(
nameof(KafkaConfig.KAFKA_EVENTS_PRODUCER_CONFIG))),
KAFKA_REQUESTS_PRODUCER_CONFIG = JsonConvert
.DeserializeObject <Dictionary <string, object> >(
Configuration.GetValue <string>(
nameof(KafkaConfig.KAFKA_REQUESTS_PRODUCER_CONFIG))),
EVENT_TOPICS = JsonConvert
.DeserializeObject <string[]>(
Configuration.GetValue <string>(
nameof(KafkaConfig.EVENT_TOPICS))),
REQUEST_TOPICS = JsonConvert
.DeserializeObject <string[]>(
Configuration.GetValue <string>(
nameof(KafkaConfig.REQUEST_TOPICS))),
SCHEMES_EVENTS_TOPIC = Configuration.GetValue <string>(
nameof(KafkaConfig.SCHEMES_EVENTS_TOPIC))
});
services.AddMemoryCache();
services.AddMvc();
}
public void SetElasticDb()
{
client = ElasticSearchConfig.GetClient();
var docs = client.Search <Document>(x => x
.Query(q => q
.MultiMatch(mp => mp
.Query(" ")
.Fields(f => f
.Fields(f1 => f1.Title, f2 => f2.Subtitle))))
.From(0)
.Size(0));
var y = 5;
}
private static void ReadConfiguration(HostBuilderContext hostContext)
{
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.AddJsonFile($"appsettings.{hostContext.HostingEnvironment.EnvironmentName}.json")
.AddEnvironmentVariables()
.Build();
ObserverConfiguration = new PricesObserverConfiguration();
configuration.GetSection("observer").Bind(ObserverConfiguration);
ElasticSearchConfig = new ElasticSearchConfig();
configuration.GetSection("ElasticSearch").Bind(ElasticSearchConfig);
}
public void Index_Exists()
{
using (var elasticsearch = new ElasticsearchInside.Elasticsearch(settings => settings
.EnableLogging()
.SetPort(9200)
.SetElasticsearchStartTimeout(180)).ReadySync())
{
ElasticSearchConfig config = new ElasticSearchConfig(new ConnectionSettings(elasticsearch.Url));
using (var indexer = new TestBaseIndex(config, new FieldDefinitionCollection(new FieldDefinition("item2", "number"))))
{
indexer.EnsureIndex(true);
indexer._client.Value.Indices.Refresh(Indices.Index(indexer.indexAlias));
Assert.IsTrue(indexer.IndexExists());
}
}
}
public ElasticSearchBaseIndex(string name,
ElasticSearchConfig connectionConfiguration,
FieldDefinitionCollection fieldDefinitions = null,
string analyzer = null,
IValueSetValidator validator = null, bool isUmbraco = false)
: base(name.ToLowerInvariant(), //TODO: Need to 'clean' the name according to Azure Search rules
fieldDefinitions ?? new FieldDefinitionCollection(), validator)
{
_connectionConfiguration = connectionConfiguration;
_isUmbraco = isUmbraco;
Analyzer = analyzer;
ElasticURL = ConfigurationManager.AppSettings[$"examine:ElasticSearch[{name}].Url"];
_searcher = new Lazy <ElasticSearchSearcher>(CreateSearcher);
_client = new Lazy <ElasticClient>(CreateElasticSearchClient);
indexAlias = prefix + Name;
tempindexAlias = indexAlias + "temp";
}
public void Rebuild_Index()
{
using (var elasticsearch = new ElasticsearchInside.Elasticsearch(settings => settings
.EnableLogging()
.SetPort(9200)
.SetElasticsearchStartTimeout(180)).ReadySync())
{
ElasticSearchConfig config = new ElasticSearchConfig(new ConnectionSettings(elasticsearch.Url));
using (var indexer = new TestBaseIndex(config, new FieldDefinitionCollection()))
{
indexer.CreateIndex();
indexer.IndexItems(indexer.AllData());
indexer._client.Value.Indices.Refresh(Indices.Index(indexer.indexAlias));
Assert.AreEqual(100, indexer.DocumentCount);
}
}
}
private void ElasticSearchConfig_ConfigChanged(object sender, EventArgs e)
{
var elasticSearchConfig = sender as ElasticSearchConfig;
if (elasticSearchConfig != null)
{
logger.Info("ElasticSearchConfig config reloading");
_config = elasticSearchConfig;
foreach (var esNode in _config.Clusters)
{
BuildCluster(esNode.ClusterName,esNode.TransportType);
ClusterThriftNodes[esNode.ClusterName] = BuildThriftNodes(esNode.ThriftNodes);
ClusterHttpNodes[esNode.ClusterName] = BuildHttpNodes(esNode.HttpNodes);
}
logger.Info("ElasticSearchConfig config reloaded");
}
else
{
logger.Error("Attempt to reload with null ElasticSearchConfig config");
}
}
private void ElasticSearchConfig_ConfigChanged(object sender, EventArgs e)
{
var elasticSearchConfig = sender as ElasticSearchConfig;
if (elasticSearchConfig != null)
{
logger.Info("ElasticSearchConfig config reloading");
_config = elasticSearchConfig;
foreach (var esNode in _config.Clusters)
{
BuildCluster(esNode.ClusterName, esNode.TransportType);
ClusterThriftNodes[esNode.ClusterName] = BuildThriftNodes(esNode.ThriftNodes);
ClusterHttpNodes[esNode.ClusterName] = BuildHttpNodes(esNode.HttpNodes);
}
logger.Info("ElasticSearchConfig config reloaded");
}
else
{
logger.Error("Attempt to reload with null ElasticSearchConfig config");
}
}
public void Can_Add_Same_Document_Twice_Without_Duplication()
{
using (var elasticsearch = new ElasticsearchInside.Elasticsearch(settings => settings
.EnableLogging()
.SetElasticsearchStartTimeout(180)).ReadySync())
{
ElasticSearchConfig config = new ElasticSearchConfig(new ConnectionSettings(elasticsearch.Url));
using (var indexer = new TestBaseIndex(config, new FieldDefinitionCollection()))
{
var value = new ValueSet(1.ToString(), "content",
new Dictionary <string, IEnumerable <object> >
{
{ "item1", new List <object>(new[] { "value1" }) },
{ "item2", new List <object>(new[] { "value2" }) }
});
indexer.IndexItem(value);
indexer.IndexItem(value);
indexer._client.Value.Indices.Refresh(Indices.Index(indexer.indexAlias));
Assert.AreEqual(1, indexer.DocumentCount);
}
}
}
private static ConnectionSettings GetConnectionSettings(ElasticSearchConfig config)
{
return(new ConnectionSettings(new Uri($"http://{config.Host}:{config.Port}")).DefaultIndex(config.DefaultIndex));
}
