static void Main(string[] args)
{
var fruits = new List <string> {
"Banana", "Kiwi", "Orange", "Apple", "Pear", "Mango", "Pineapple"
};
var rand = new Random();
var client = new ElasticsearchClient();
while (true)
{
var fruit = new FruitDelivery()
{
Id = Guid.NewGuid().ToString(),
Name = fruits[rand.Next(0, fruits.Count())],
Quantity = rand.Next(1, 20)
};
var indexResponse = client.Index("fruit", "fruit", fruit.Id, fruit);
JsonLogger.LogObject(new { FruitName = fruit.Name, Quantity = fruit.Quantity, ClientName = "FruitDelivery" });
Thread.Sleep(100);
}
}
static void Main(string[] args)
{
var client = new ElasticsearchClient();
var stopwatch = new Stopwatch();
stopwatch.Start();
int totalCount = 100000;
for (int i = 0; i < totalCount; i++)
{
var creditCard = new CreditCard();
creditCard.Buyer = new Buyer();
creditCard.InstantBuyKey = Guid.NewGuid();
creditCard.Brand = i % 2 == 0 ? "Visa" : "Mastercard";
creditCard.Buyer.BuyerKey = Guid.NewGuid();
creditCard.Buyer.Name = Guid.NewGuid().ToString();
client.Index("mundipagg", "creditcards", creditCard.InstantBuyKey.ToString(), creditCard);
}
stopwatch.Stop();
decimal avg = stopwatch.ElapsedMilliseconds / totalCount;
Console.WriteLine();
Console.WriteLine("TOTAL: " + stopwatch.ElapsedMilliseconds + " ms");
Console.WriteLine("AVG: " + avg + " ms / item");
Console.ReadLine();
//var indexResponse = client.Index("myindex", "mytype", "1", new { Hello = "World" });
}
public static ElasticsearchClient GetOrAddClient <TConfig>(
this IEphemeralCluster <TConfig> cluster,
Func <ElasticsearchClientSettings, ElasticsearchClientSettings> modifySettings = null
)
where TConfig : EphemeralClusterConfiguration
{
modifySettings ??= s => s;
return(cluster.GetOrAddClient(c =>
{
var settings = modifySettings(cluster.CreateConnectionSettings());
var current = (ITransportClientConfigurationValues)settings;
var notAlreadyAuthenticated = current.Authentication == null &&
current.ClientCertificates == null;
var noCertValidation = current.ServerCertificateValidationCallback == null;
if (cluster.ClusterConfiguration.EnableSecurity && notAlreadyAuthenticated)
{
settings = settings.Authentication(new BasicAuthentication(ClusterAuthentication.Admin.Username,
ClusterAuthentication.Admin.Password));
}
if (cluster.ClusterConfiguration.EnableSsl && noCertValidation)
{
//todo use CA callback instead of allowall
// ReSharper disable once UnusedVariable
var ca = new X509Certificate2(cluster.ClusterConfiguration.FileSystem.CaCertificate);
settings = settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll);
}
var client = new ElasticsearchClient(settings);
return client;
}));
}
/// <summary>
/// Construct a sink posting to the specified Elasticsearch cluster.
/// </summary>
/// <param name="connectionConfiguration">Connection configuration to use for connecting to the cluster.</param>
/// <param name="indexFormat">The index name formatter. A string.Format using the DateTime.UtcNow is run over this string.</param>
/// <param name="batchPostingLimit">The maximum number of events to post in a single batch.</param>
/// <param name="period">The time to wait between checking for event batches.</param>
/// <param name="formatProvider">Supplies culture-specific formatting information, or null.</param>
public ElasticSearchSink(ConnectionConfiguration connectionConfiguration, string indexFormat, int batchPostingLimit, TimeSpan period, IFormatProvider formatProvider)
: base(batchPostingLimit, period)
{
_indexFormat = indexFormat;
_formatProvider = formatProvider;
_client = new ElasticsearchClient(connectionConfiguration);
}
protected override async Task <ApiResponse> InternalExecuteAsync(object request)
{
string url = string.Format("{0}{1}", request, Request.Url.Query);
var es = new ElasticsearchClient(AppSettings.Current.ClusterUrl);
JsonObject jo = null;
switch (Request.Method)
{
case "GET":
break;
case "POST":
var body = RequestBody();
jo = string.IsNullOrWhiteSpace(body) ? null : JsonObject.Parse(body);
break;
default:
throw new Exception(string.Format("Invalid HTTP methos [{0}]", Request.Method));
}
var response = await es.SearchAsync(new SearchRequest(url, jo));
return(new ApiResponse(response.ToString(), (HttpStatusCode)(int)response.StatusCode));
}
public async Task RemovesAliasFromOldIndexAndAddsToNewIndex()
{
CreateIndexWithConfig("initialindex", "{}");
var alias = _fixture.Create <string>();
await AssignToAlias(alias, "initialindex");
var sqsMessage = new SqsMessage();
_sqsClientMock.Setup(s => s.SendMessageAsync(It.Is <SendMessageRequest>(
s => StoreMessage(s, out sqsMessage)), default))
.ReturnsAsync(new SendMessageResponse());
await _classUnderTest.ReindexAlias(new ReindexRequest { alias = alias }, null);
sqsMessage.alias.Should().Be(alias);
System.Threading.Thread.Sleep(1000);
sqsMessage.timeCreated = DateTime.Now.AddSeconds(-600);
await _classUnderTest.SwitchAlias(SqsEvent(sqsMessage), null);
var indices = await ElasticsearchClient.GetIndicesPointingToAliasAsync(alias);
indices.Count.Should().Be(1);
indices.First().Should().Contain(sqsMessage.newIndex);
}
/**
* A custom serializer would not know how to serialize `QueryContainer` or other NEST types that could appear as part of
* the `_source` of a document, therefore a custom serializer needs to have a way to delegate serialization of NEST types
* back to NEST's built-in serializer.
*/
/**
* ==== JsonNetSerializer
*
* We ship a separate {nuget}/NEST.JsonNetSerializer[NEST.JsonNetSerializer] package that helps in composing a custom `SourceSerializer`
* using `Json.NET`, that is smart enough to delegate the serialization of known NEST types back to the built-in
* `RequestResponseSerializer`. This package is also useful if
*
* . You want to control how your documents and values are stored and retrieved from Elasticsearch using `Json.NET`
* . You want to use `Newtonsoft.Json.Linq` types such as `JObject` within your documents
*
* The easiest way to hook this custom source serializer up is as follows
*/
public void DefaultJsonNetSerializer()
{
var pool = new SingleNodePool(new Uri("http://localhost:9200"));
var connectionSettings =
new ElasticsearchClientSettings(pool, sourceSerializer: JsonNetSerializer.Default);
var client = new ElasticsearchClient(connectionSettings);
}
public SearchTemplateRepository(string clusterUrl, string index, ISimpleCache <SearchTemplate> cache, CacheExpiration expiration)
{
_es = new ElasticsearchClient(clusterUrl);
_index = index;
_cache = cache;
_expiration = expiration;
}
public SearchTemplateRepository(CacheExpiration expiration)
{
_es = new ElasticsearchClient(DEFAULT_CLUSTER);
_index = DEFAULT_INDEX;
_cache = new SearchTemplateCache(CleanupFireAndForget);
_expiration = expiration;
}
protected override void IntegrationSetup(ElasticsearchClient client, CallUniqueValues values)
{
foreach (var id in values.Values)
{
Client.Index(Project.Instance, i => i.Id(id).Routing(id));
}
}
public static async Task <GetResponse <T> > Get <T>(this ElasticsearchClient client,
string id, string typeName, string indexName = null) where T : new()
{
var url = (indexName ?? client.DefaultIndexName) + "/" + typeName + "/" + id;
return(await client.Execute <GetResponse <T> >(RestMethod.GET, url));
}
public void CanHandle_ElasticsearchClient_True()
{
var client = new ElasticsearchClient(Option);
var factory = new ElasticsearchCommandFactory();
Assert.That(factory.CanHandle(client), Is.True);
}
public static ElasticsearchResponse <DynamicDictionary> IndexAlbum(string url)
{
var uri = new Uri(url);
var config = new ConnectionConfiguration(uri);
config.ExposeRawResponse(true);
var client = new ElasticsearchClient(config);
// 'dynamically'-typed
Console.WriteLine(" --- 'dynamically'-typed --- ");
ElasticsearchResponse <DynamicDictionary> response = client.Index("rolling-stone-500", "album", "1", Album.ChronicleV1);
Console.WriteLine("_index: " + response.Response["_index"]);
Console.WriteLine("_type: " + response.Response["_type"]);
Console.WriteLine("_id: " + response.Response["_id"]);
Console.WriteLine("_version: " + response.Response["_version"]);
Console.WriteLine("created: " + response.Response["created"]);
// strongly-typed
Console.WriteLine(" --- strongly-typed --- ");
ElasticsearchResponse <Album> response2 = client.Index <Album>("rolling-stone-500", "album", "2", Album.ChronicleV1);
//Console.WriteLine("_index: " + response2.Response._index);
//Console.WriteLine("_type: " + response2.Response._type);
//Console.WriteLine("_id: " + response2.Response._id);
//Console.WriteLine("_version: " + response2.Response._version);
// Console.WriteLine("created: " + response2.Response.created); // removed this because it's getting indexed as well -- only want it to pick up the response
// show raw response (requires config on connection)
Console.WriteLine(" --- raw response --- ");
Console.WriteLine(Encoding.UTF8.GetString(response2.ResponseRaw));
return(response);
}
/**
* This however is still a non-failover connection. Meaning if that `node` goes down the operation will not be retried on any other nodes in the cluster.
*
* To get a failover connection we have to pass an `IConnectionPool` instance instead of a `Uri`.
*/
public void InstantiatingAConnectionPoolClient()
{
var node = new Uri("http://mynode.example.com:8082/apiKey");
var connectionPool = new SniffingConnectionPool(new[] { node });
var config = new ConnectionConfiguration(connectionPool);
var client = new ElasticsearchClient(config);
}
public void PostFood(Food food)
{
var ESclient = new ElasticsearchClient();
var response = ESclient.Index <Food>("food", food.Type, food.Id, food);
redDB.StringSet(food.Id, food.Name);
}
static void Main(string[] args)
{
RiverConfiguration _riverConfiguration = (RiverConfiguration)ConfigurationManager.GetSection("riverConfiguration");
ConnectionSettings searchSettings = new ConnectionSettings(new Uri(_riverConfiguration.ElasticSearchConnectionString));
var searchClient = new ElasticsearchClient(searchSettings);
while (true)
{
//Connect to DB and listen for a message.
using (SqlConnection sqlConnection = new SqlConnection(ConfigurationManager.ConnectionStrings["queueDatabase"].ConnectionString))
{
sqlConnection.Open();
SqlCommand receiveCommand = new SqlCommand("WAITFOR( RECEIVE TOP(1) CONVERT(XML, message_body) AS Message FROM RiverUpdateReceiveQueue )", sqlConnection);
SqlCommand retrieveDataCommand;
RiverMessage riverMessage;
using (SqlDataReader receiveCommandReader = receiveCommand.ExecuteReader())
{
//Deserialize message.
receiveCommandReader.Read();
XmlSerializer serializer = new XmlSerializer(typeof(RiverMessage));
riverMessage = (RiverMessage)serializer.Deserialize(receiveCommandReader.GetXmlReader(0));
//Get the entire record out of the DB.
retrieveDataCommand = new SqlCommand(string.Format("SELECT TOP(1) * FROM {0} WHERE Id = {1}", riverMessage.DatabaseTable, riverMessage.Id), sqlConnection);
}
using (SqlDataReader retrieveDataCommandReader = retrieveDataCommand.ExecuteReader())
{
JObject item = null;
//Read it from the DB and store in a simple JSON object.
if (retrieveDataCommandReader.Read())
{
item = new JObject();
for (int i = 0; i < retrieveDataCommandReader.FieldCount; i++)
{
item[retrieveDataCommandReader.GetName(i)] = new JValue(retrieveDataCommandReader.GetValue(i));
}
}
//Foreach river that wants this type of record. Send it.
foreach (var riverRequired in _riverConfiguration.Rivers.Cast <RiverElement>().Where(x => x.DatabaseTable == riverMessage.DatabaseTable))
{
if (item != null)
{
searchClient.Index(riverRequired.ElasticIndex, riverRequired.ElasticType, riverMessage.Id.ToString(), item.ToString());
}
else
{
searchClient.Delete(riverRequired.ElasticIndex, riverRequired.ElasticType, riverMessage.Id.ToString());
}
}
}
}
}
}
private async Task IndexDocuments(string index, IEnumerable <QueryableAddress> documents)
{
foreach (var document in documents)
{
var indexed = await ElasticsearchClient.IndexAsync(document,
i => i.Index(index).Refresh(Refresh.WaitFor));
}
}
public async Task <string> SearchAllAsync(string query, int @from, int size)
{
var client = new ElasticsearchClient(_settings);
var queryString = "{\"from\" : " + from + ", \"size\" : " + size + ", \"query\":{\"match\": {\"name\": {\"query\": \" " + query + " \",\"operator\": \"and\"}}}}";
var res = await client.SearchAsync <string>(Setting.ElasticSearchIndex, queryString);
return(res.Response);
}
/// <summary>
/// Initializes a new instance of the <see cref="Common.ElasticSearch"/> class.
/// Follow this: http://nest.azurewebsites.net/elasticsearch-net/connecting.html
/// We use a ConnectionPool to make the connection fail-over, that means, if the
/// connection breaks, it reconnects automatically
/// </summary>
private ElasticSearch()
{
var node = new Uri(connectionString);
var connectionPool = new SniffingConnectionPool(new[] { node });
var config = new ConnectionConfiguration(connectionPool);
client = new ElasticsearchClient(config); // exposed in this class
}
public async Task Cluster_must_return_correct_version()
{
var es = new ElasticsearchClient();
var cr = await es.GetClusterAsync();
Assert.IsTrue(cr.Name == "elasticsearch");
Assert.IsTrue(cr.VersionNumber > 6);
}
/**
* Hooking up the serializer is performed in the `ConnectionSettings` constructor
*/
public void TheNewContract()
{
var pool = new SingleNodePool(new Uri("http://localhost:9200"));
var connectionSettings = new ElasticsearchClientSettings(
pool,
sourceSerializer: (builtin, settings) => new VanillaSerializer()); // <1> what the Func?
var client = new ElasticsearchClient(connectionSettings);
}
public static void ShowInfo(ElasticsearchClient client)
{
//------------------------
var response = client.Info <string>();
Console.WriteLine(response.Response);
//------------------------
}
/// <summary>
/// Shortcut into the Bulk call that indexes the specified objects
/// <para> </para>
/// https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
/// </summary>
/// <param name="client"></param>
/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
/// <param name="objects">List of objects to index, Id will be inferred (Id property or IdProperty attribute on type)</param>
/// <param name="index">Override the inferred indexname for T</param>
/// <param name="type">Override the inferred typename for T</param>
public static Task<BulkResponse> IndexManyAsync<T>(this ElasticsearchClient client, IEnumerable<T> objects, IndexName index = null,
CancellationToken cancellationToken = default
)
where T : class
{
var bulkRequest = CreateIndexBulkRequest(objects, index);
return client.BulkAsync(bulkRequest, cancellationToken);
}
public async Task Alias_must_map_correctly()
{
var es = new ElasticsearchClient();
await es.DeleteIndexAsync("test");
var cir = await es.CreateIndexAsync("test");
Assert.IsTrue(cir.IsSuccess);
Assert.IsTrue(cir.Index == "test");
var source = new JsonObject()
.Add("name", "YM");
var document = new ElasticsearchDocument("1", source, "test");
var idr = await es.IndexDocumentAsync(new IndexDocumentRequest(document, true));
var search = new SearchRequest("test")
.SetQuery(new MatchQuery("name", "YM"));
var hits = await es.SearchAsync(search);
Assert.IsTrue(hits.Hits.Total == 1);
var hit = hits.Hits.Hits[0];
Assert.IsTrue(hit.Id == "1");
var response = await es.AliasAsync(new AliasRequest().Add("test", "test-alias"));
Assert.IsTrue(response.IsSuccess);
var aliases = await es.GetIndexAliasesAsync("test");
Assert.IsTrue(aliases.Aliases[0] == "test-alias");
var indices = await es.GetAliasIndicesAsync("test-alias");
Assert.IsTrue(indices.Indices[0] == "test");
search = new SearchRequest("test-alias")
.SetQuery(new MatchQuery("name", "YM"));
hits = await es.SearchAsync(search);
Assert.IsTrue(hits.Hits.Total == 1);
hit = hits.Hits.Hits[0];
Assert.IsTrue(hit.Id == "1");
response = await es.AliasAsync(new AliasRequest().Remove("test", "test-alias"));
Assert.IsTrue(response.IsSuccess);
search = new SearchRequest("test-alias")
.SetQuery(new MatchQuery("name", "YM"));
hits = await es.SearchAsync(search);
Assert.IsNull(hits.Hits);
}
public void Instantiate_ElasticsearchClientAndQuery_CommandNotNull()
{
var client = new ElasticsearchClient(Option);
var query = Mock.Of <IQuery>(x => x.Statement == @"GET index/type/_search { ""query"": {""match_all"": { }} }");
var factory = new ElasticsearchCommandFactory();
var command = factory.Instantiate(client, query);
Assert.That(command, Is.Not.Null);
}
public async Task <string> SearchIngredientsAsync(string query, int @from, int size)
{
var client = new ElasticsearchClient(_settings);
var queryString = "{\"from\": " + from + ", \"size\": " + size + ", \"filter\": { \"or\": [{\"term\": { \"dataType\": \"hop\"}},{\"term\": {\"dataType\": \"fermentable\"}},{\"term\": {\"dataType\": \"yeast\"}},{\"term\": {\"dataType\": \"other\"}}]},\"query\": {\"match\": {\"name\": {\"query\": \"" + query + "\"}}}}";
var res = await client.SearchAsync <string>(Setting.ElasticSearchIndex, queryString);
return(res.Response);
}
public async Task Search_request_must_transform_to_correct_ES_request()
{
var es = new ElasticsearchClient();
await es.DeleteIndexAsync("test");
var cir = await es.CreateIndexAsync("test");
Assert.IsTrue(cir.IsSuccess);
Assert.IsTrue(cir.Index == "test");
var documents = new ElasticsearchDocument[] {
new ElasticsearchDocument("1", new JsonObject().Add("id", 1).Add("name", "YM"), "test"),
new ElasticsearchDocument("2", new JsonObject().Add("id", 2).Add("name", "SV"), "test")
};
var br = await es.BulkDocumentsAsync(new BulkRequest(true).Index(documents));
Assert.IsTrue(br.IsSuccess);
var search = new SearchRequest("test")
.SetQuery(new MatchQuery("name", "YM"));
var hits = await es.SearchAsync(search);
Assert.IsTrue(hits.Hits.Total == 1);
var hit = hits.Hits.Hits[0];
Assert.IsTrue(hit.Id == "1");
Assert.IsTrue(hit.Source.Property <int>("id") == 1);
search = new SearchRequest("test")
.SetQuery(new TermQuery("id", 2));
hits = await es.SearchAsync(search);
Assert.IsTrue(hits.Hits.Total == 1);
hit = hits.Hits.Hits[0];
Assert.IsTrue(hit.Id == "2");
Assert.IsTrue(hit.Source.Property <string>("name") == "SV");
search = new SearchRequest("test")
.SetQuery(new ConstantScoreQuery(new RangeQuery("id", 1, null)))
.SetFrom(1)
.SetSize(1)
.HideSource();
hits = await es.SearchAsync(search);
Assert.IsTrue(hits.Hits.Total == 2);
Assert.IsTrue(hits.Hits.Hits.Length == 1);
hit = hits.Hits.Hits[0];
Assert.IsTrue(hit.Score == 1);
Assert.IsNull(hit.Source);
}
public SyncLock(ElasticsearchClient client, string index, string type, bool force = false)
{
Client = client;
LockIndex = index;
LockType = type;
Force = force;
Open();
}
public async Task UnsuccessfulGetAsyncShouldThrowException()
{
var(factory, client) = GetMocks();
var response = ExceptionGetResponse <TestEntity>(new ElasticsearchClientException("Smth went wrong"));
client.Setup(x => x.GetAsync(It.IsAny <DocumentPath <TestEntity> >(), null, It.IsAny <CancellationToken>()))
.ReturnsAsync(response.Object);
var elasticSearchClient = new ElasticsearchClient(factory.Object);
await Assert.ThrowsAsync <ElasticsearchException>(() => elasticSearchClient.GetAsync <TestEntity>("id", "index"));
}
public async Task UnsuccessfulSearchAsyncShouldThrowException()
{
var(factory, client) = GetMocks();
var response = UnsuccessfulSearchResponse <TestEntity>("smth went wrong");
client.Setup(x => x.SearchAsync <TestEntity>(It.IsAny <ISearchRequest>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(response.Object);
var elasticSearchClient = new ElasticsearchClient(factory.Object);
await Assert.ThrowsAsync <ElasticsearchException>(() => elasticSearchClient.SearchAsync <TestEntity>("Name", "John", "index"));
}
static YamlTestsBase()
{
var host = "localhost";
if (Process.GetProcessesByName("fiddler").Any())
host = "ipv4.fiddler";
var uri = new Uri("http://"+host+":9200/");
var settings = new ConnectionConfiguration(uri).UsePrettyResponses();
_client = new ElasticsearchClient(settings);
var infoResponse = _client.Info();
dynamic info = infoResponse.Response;
_versionNumber = new Version(info.version.number);
}
public void SniffCalledOnceAndEachEnpointPingedOnce()
{
using (var fake = new AutoFake(callsDoNothing: true))
{
//It's recommended to only have on instance of your connection pool
//Be sure to register it as Singleton in your IOC
var uris = new[] { new Uri("http://localhost:9200"), new Uri("http://localhost:9201") };
var connectionPool = new SniffingConnectionPool(uris);
var config = new ConnectionConfiguration(connectionPool)
.SniffOnStartup();
fake.Provide<IConnectionConfigurationValues>(config);
var pingCall = FakeCalls.PingAtConnectionLevel(fake);
pingCall.Returns(FakeResponse.Ok(config));
var sniffCall = FakeCalls.Sniff(fake, config, uris);
var getCall = FakeCalls.GetSyncCall(fake);
getCall.Returns(FakeResponse.Ok(config));
var transport1 = FakeCalls.ProvideRealTranportInstance(fake);
var transport2 = FakeCalls.ProvideRealTranportInstance(fake);
var transport3 = FakeCalls.ProvideRealTranportInstance(fake);
var transport4 = FakeCalls.ProvideRealTranportInstance(fake);
transport1.Should().NotBe(transport2);
var client1 = new ElasticsearchClient(config, transport: transport1);
client1.Info();
client1.Info();
client1.Info();
client1.Info();
var client2 = new ElasticsearchClient(config, transport: transport2);
client2.Info();
client2.Info();
client2.Info();
client2.Info();
var client3 = new ElasticsearchClient(config, transport: transport3);
client3.Info();
client3.Info();
client3.Info();
client3.Info();
var client4 = new ElasticsearchClient(config, transport: transport4);
client4.Info();
client4.Info();
client4.Info();
client4.Info();
sniffCall.MustHaveHappened(Repeated.Exactly.Once);
//sniff validates first node, one new node should be pinged before usage.
pingCall.MustHaveHappened(Repeated.Exactly.Once);
}
}
public void SniffOnStartupCallsSniffOnlyOnce()
{
using (var fake = new AutoFake(callsDoNothing: true))
{
//It's recommended to only have on instance of your connection pool
//Be sure to register it as Singleton in your IOC
var uris = new[] { new Uri("http://localhost:9200") };
var connectionPool = new SniffingConnectionPool(uris);
var config = new ConnectionConfiguration(connectionPool)
.DisablePing()
.SniffOnStartup();
fake.Provide<IConnectionConfigurationValues>(config);
var sniffCall = FakeCalls.Sniff(fake, config, uris);
var transport = FakeCalls.ProvideDefaultTransport(fake);
var client1 = new ElasticsearchClient(config, transport: transport);
var client2 = new ElasticsearchClient(config, transport: transport);
var client3 = new ElasticsearchClient(config, transport: transport);
var client4 = new ElasticsearchClient(config, transport: transport);
sniffCall.MustHaveHappened(Repeated.Exactly.Once);
}
}
public ElasticSearchNetClient(ElasticsearchClient client, string index, string type) {
Client = client;
Index = index;
Type = type;
}
