public async Task IndexEntryAsync(T retrievedInfo)
{
var asyncIndexResponse = await lowlevelClient
.IndexAsync <StringResponse>(index, mapping, PostData.Serializable(retrievedInfo));
var response = asyncIndexResponse.Body;
}
public Task Log(MonitoringLogEntry logEntry)
{
ThreadPool.QueueUserWorkItem(async x => {
var response = await _client.IndexAsync <StringResponse>("monitoring", logEntry.GetType().Name, logEntry.Id.ToString(), PostData.Serializable(logEntry));
CheckReponseSuccess(response, "monitoring");
});
return(Task.Delay(0));
}
/// <summary>
///
/// </summary>
/// <param name="index">index的名字类似数据库名或者表名?</param>
/// <param name="id">id</param>
/// <param name="document">相当于一个对象</param>
/// <returns></returns>
public static async Task <bool> IndexAsync(string index, string id, object document)
{
var syncIndexResponse =
await _lowlevelClient.IndexAsync <StringResponse>(index, id, PostData.Serializable(document));
return(syncIndexResponse.Success);
}
/// <summary>
/// Method: ElasticPut
/// Goal: Sends data to the ElasticSearch Cluster
/// </summary>
/// <param name="lowlevelClient">The ElasticSearch LowLevelClient Object, returned after the connection is made</param>
/// <param name="jsonLogEntry">The emp_json_log_entry object entry with the representation of the log message</param>
/// <param name="log">The ILogger object to log information and errors</param>
/// <returns></returns>
/// <exception cref="UnexpectedElasticsearchClientException">Unknown exceptions, for instance a response from Elasticsearch not properly deserialized. These are sometimes bugs and should be reported.</exception>
/// <exception cref="ElasticsearchClientException">
/// Known exceptions that includes: max retries or timeout reached, bad authentication, etc.
/// Elasticsearch itself returned an error (could not parse the request, bad query, missing field, etc.
///</exception>
public static async Task ElasticPutAsync(ElasticLowLevelClient lowlevelClient, JsonLogEntry jsonLogEntry, ILogger log)
{
try
{
string indexString;
DateTime dt = Convert.ToDateTime(jsonLogEntry.Date);
string strYear = (dt.Year.ToString());
indexString = $"apps_{jsonLogEntry.Trigram.ToLower()}_{strYear}";
var asyncIndexResponse = await lowlevelClient.IndexAsync <StringResponse>(indexString, PostData.Serializable(jsonLogEntry));
string indexResponse = asyncIndexResponse.Body;
var success = asyncIndexResponse.Success;
if (success)
{
log?.LogInformation($"ElasticPut: Log entry successfully sent to Elastic Cluster {asyncIndexResponse.Body}");
}
else
{
log?.LogInformation($"ElasticPut: An exception was thrown by the Elastic method {asyncIndexResponse.OriginalException}");
throw new Exception("Could not write in elastic");
}
}
catch (UnexpectedElasticsearchClientException e)
{
log?.LogError($"ElasticPut: UnexpectedElasticsearchClientException : {e.Message}");
throw e;
}
catch (ElasticsearchClientException e)
{
log?.LogError($"ElasticPut: ElasticsearchClientException : {e.Message}");
throw e;
}
}
static async Task Main(string[] args)
{
// var lowlevelClient = new ElasticLowLevelClient();
var settings = new ConnectionConfiguration(new Uri("http://localhost:9200")).RequestTimeout(TimeSpan.FromMinutes(2));
var lowlevelClient = new ElasticLowLevelClient(settings);
// var uris = new []
// {
// new Uri("http://es01:9200"),
// new Uri("http://es02:9201"),
// new Uri("http://es03:9202"),
// };
// var connectionPool = new SniffingConnectionPool(uris);
// var settings = new ConnectionConfiguration(connectionPool);
// var lowlevelClient = new ElasticLowLevelClient(settings);
var person = new Person
{
FirstName = "Will",
LastName = "Laarman"
};
var ndexResponse = lowlevelClient.Index <BytesResponse>("people", "1", PostData.Serializable(person));
byte[] responseBytes = ndexResponse.Body;
Console.WriteLine();
Console.WriteLine(System.Text.Encoding.UTF8.GetString(responseBytes));
Console.WriteLine();
var asyncIndexResponse = await lowlevelClient.IndexAsync <StringResponse>("people", "1", PostData.Serializable(person));
string responseString = asyncIndexResponse.Body;
var searchResponse = lowlevelClient.Search <StringResponse>("people", PostData.Serializable(new
{
from = 0,
size = 10,
query = new
{
match = new
{
firstName = new {
query = "Will"
}
}
}
}));
var successful = searchResponse.Success;
var successOrKnownError = searchResponse.SuccessOrKnownError;
var exception = searchResponse.OriginalException;
var responseJson = searchResponse.Body;
Console.WriteLine(responseJson);
}
private async Task GatherBTCAsync(CancellationToken cancellationToken)
{
var resultBTC = await Client.GetAsync("https://api.bitfinex.com/v2/ticker/tBTCUSD");
var test = new PricesResponse();
if (resultBTC.IsSuccessStatusCode)
{
// var response = JsonConvert.DeserializeObject<PricesResponse>(resultETH.Content.ReadAsStringAsync().Result);
var response = resultBTC.Content.ReadAsStringAsync().Result;
var data = Array.ConvertAll(response.Trim('[', ']').Split(','), s => float.Parse(s));
var ticker = ParseTicker(data);
var asyncIndexResponseBTC = await EsClient.IndexAsync <string>("btc", "Currency", Guid.NewGuid().ToString(), ticker);
string responseString = asyncIndexResponseBTC.Body;
}
}
public async Task StoreAsync(LogContext logContext, CancellationToken cancellationToken = default)
{
var response = await _client.IndexAsync <BytesResponse>(_options.Index, logContext.TraceId, PostData.Serializable(new ElasticLogContext(logContext)), ctx : cancellationToken);
if (!response.Success)
{
_logger.LogError(response.OriginalException, response.OriginalException.Message);
}
}
public async Task <bool> Send(string entry)
{
StringResponse result = await _client.IndexAsync <StringResponse>(_index, PostData.String(entry));
if (result.HttpStatusCode is >= 200 and < 300)
{
return(true);
}
return(false);
}
static async System.Threading.Tasks.Task Main(string[] args)
{
var settings = new ConnectionConfiguration(new Uri("http://localhost:9200/"))
.RequestTimeout(TimeSpan.FromMinutes(2));
var client = new ElasticLowLevelClient(settings);
var cars = new Cars("Ford", 1000000);
//var cars = new Cars("Ford", 200000);
var response = await client.IndexAsync <StringResponse>("cars", "1", PostData.Serializable(cars));
string responseString = response.Body;
Console.WriteLine(responseString);
}
internal void IndexEvent(TelemetryEventBase plainEvent)
{
try
{
//Check if we can connect to the ElasticSearch Server
if (_ElasticClient != null &&
!_ElasticClient.IndexAsync <BytesResponse>(_ElasticIndex, _ElasticType, PostData.Serializable(plainEvent)).Result.Success)
{
logger.Error("Server settings are not configured");
_ElasticClient = null;
return;
}
}
catch (Exception e) { logger.Fatal(e); }
}
public static void IndexLab(ElasticLowLevelClient lowlevelClient)
{
var person = new Person
{
Id = 1, Name = "Jack"
};
var indexResponse = lowlevelClient.Index <BytesResponse>("people", "person", "1", PostData.Serializable(person));
byte[] responseBytes = indexResponse.Body;
var asyncIndexResponse = lowlevelClient.IndexAsync <StringResponse>("people", "person", "1", PostData.Serializable(person)).GetAwaiter().GetResult();
string responseString = asyncIndexResponse.Body;
Console.WriteLine(responseString);
}
static async Task Main(string[] args)
{
var settings = new ConnectionConfiguration(new Uri("http://localhost:9200")).RequestTimeout(TimeSpan.FromMinutes(2));
var lowlevelClient = new ElasticLowLevelClient(settings);
var person = new
{
FirstName = "liu",
LastName = "jie",
Age = "18"
};
//var indexResponse = lowlevelClient.Index<BytesResponse>("people", "1", PostData.Serializable(person));
//byte[] responseBytes = indexResponse.Body;
//异步的方式
//如果不指定id，那么会每次添加都会自动生成一个id，就算是相同的数据，指定了的话，如果原先没有doc就会创建，已有的话就会update
var asyncIndexResponse = await lowlevelClient.IndexAsync <StringResponse>("people", "2", PostData.Serializable(person));
string responseString = asyncIndexResponse.Body;
Console.WriteLine(responseString);
Console.ReadKey();
//search 搜索
var searchResponse = lowlevelClient.Search <StringResponse>("people", PostData.Serializable(new
{
from = 0,
size = 10,
query = new
{
match = new
{
LastName = new
{
query = "jie"
}
}
}
}));
var successful = searchResponse.Success;
var responseJson = searchResponse.Body;
Console.WriteLine(successful);
Console.WriteLine(responseJson);
Console.ReadKey();
}
public async void AddDocToEsTest()
{
string esInstance = Environment.GetEnvironmentVariable("ES_INSTANCE");
var settings = new ConnectionConfiguration(new Uri(esInstance))
.RequestTimeout(TimeSpan.FromMinutes(2));
var lowlevelClient = new ElasticLowLevelClient(settings);
var sample_guid = new Guid
{
guid = "Martijn"
};
var asyncIndexResponse = await lowlevelClient.IndexAsync <StringResponse>("test_post", "1", PostData.Serializable(sample_guid));
string responseString = asyncIndexResponse.Body;
Console.WriteLine(responseString);
}
private async void cmdInsertAsync_Click(object sender, EventArgs e)
{
var settings = new ConnectionConfiguration(EndPoint())
.RequestTimeout(TimeSpan.FromMinutes(2));
var lowlevelClient = new ElasticLowLevelClient(settings);
var person = new Person
{
firstName = "Rebecca",
lastName = "Betteridge"
};
var asyncIndexResponse = await lowlevelClient.IndexAsync <StringResponse>("people", "person", "Rebecca", PostData.Serializable(person));
var responseString = asyncIndexResponse.Body;
txtResult.Text = PrettyPrintJSON(responseString);
}
private async Task UpsertAsync(NamedId <DomainId> appId, NamedId <DomainId> schemaId, UpsertIndexEntry upsert)
{
var data = new
{
appId = appId.Id,
appName = appId.Name,
contentId = upsert.ContentId,
schemaId = schemaId.Id,
schemaName = schemaId.Name,
serveAll = upsert.ServeAll,
servePublished = upsert.ServePublished,
texts = upsert.Texts
};
var result = await client.IndexAsync <StringResponse>(IndexName, upsert.DocId, CreatePost(data));
if (!result.Success)
{
throw new InvalidOperationException($"Failed with ${result.Body}", result.OriginalException);
}
}
public async System.Threading.Tasks.Task <IActionResult> LowLevelNewIndexAsync()
{
// Connecting using additional settings
var settings = new ConnectionConfiguration(new Uri("http://localhost:9200/"))
.RequestTimeout(TimeSpan.FromMinutes(2));
var lowlevelClient = new ElasticLowLevelClient(settings);
// Indexing
var person = new
{
firstName = "Ignacio",
lastName = "Rognoni"
};
// Async method
var asyncIndexResponse = await lowlevelClient.IndexAsync <StringResponse>("people", "1", PostData.Serializable(person));
string responseString = asyncIndexResponse.Body;
return(Ok(responseString));
}
/// <summary>
/// This method is called for every Lambda invocation. This method takes in an S3 event object and can be used
/// to respond to S3 notifications.
/// </summary>
/// <param name="evnt"></param>
/// <param name="context"></param>
/// <returns></returns>
public async Task <bool> FunctionHandler(S3Event evnt, ILambdaContext context)
{
//setting up configuration object (use if required)
// var builder = new ConfigurationBuilder()
//.SetBasePath(Directory.GetCurrentDirectory())
//.AddJsonFile("appsettings.json");
// Configuration = builder.Build();
//getting the s3event
var s3Event = evnt.Records?[0].S3;
if (s3Event == null)
{
return(false);
}
try
{
//setting up elasticsearch search client to save the logs
var node = new Uri("[elasticserach endpoint goes here]");
var config = new ConnectionConfiguration(node);
var esClient = new ElasticLowLevelClient(config);
var response = await S3Client.GetObjectAsync(s3Event.Bucket.Name, s3Event.Object.Key);
//this string read all the content from s3 object
string responseString;
using (var reader = new StreamReader(response.ResponseStream))
{
responseString = reader.ReadToEnd();
}
//get array of the logs by spliting with new line
string[] allRequests = responseString.Split(new string[] { "\r\n", "\n" }, StringSplitOptions.None);
//this will keep track of number of logs added for tracking
int processCount = 0;
foreach (var request in allRequests)
{
//reading all the log meta data - they in format below
/*timestamp
* elb
* client:port
* backend:port
* request_processing_time backend_processing_time
* response_processing_time
* elb_status_code
* backend_status_code
* received_bytes
* sent_bytes
* "request"
* "user_agent"
* ssl_cipher ssl_protocol*/
var allParams = request.Split(new[] { " " }, StringSplitOptions.None);
var doc = new Document();
for (int i = 0; i < allParams.Length - 1; i++)
{
if (i == 0)
{
doc.RequestTime = Convert.ToDateTime(allParams[i]).ToString("yyyy-MM-dd");
}
else if (i == 1)
{
doc.Elb = allParams[i];
}
else if (i == 2)
{
doc.ClientPort = allParams[i];
}
else if (i == 3)
{
doc.BackendPort = allParams[i];
}
else if (i == 4)
{
doc.RequestProcessingTime = allParams[i];
}
else if (i == 5)
{
doc.BackendProcessingTime = allParams[i];
}
else if (i == 6)
{
doc.ResponseProcessingTime = allParams[i];
}
else if (i == 7)
{
doc.ElbStatusCode = allParams[i];
}
else if (i == 8)
{
doc.BackendStatusCode = allParams[i];
}
else if (i == 9)
{
doc.ReceivedBytes = allParams[i];
}
else if (i == 10)
{
doc.SentBytes = allParams[i];
}
else if (i == 11)
{
doc.RequestType = allParams[i].Replace("\"", "");
}
else if (i == 12)
{
doc.RequestUrl = allParams[i];
}
}
var jsonstring = JsonConvert.SerializeObject(doc);
//adding the record to elastic search
await esClient.IndexAsync <object>("logs", "searchapielb", jsonstring);
processCount++;
}
Console.WriteLine("Added " + processCount + " number of logs");
//clean up delete the s3 object added (apply only if required)
await S3Client.DeleteObjectAsync(new DeleteObjectRequest
{
BucketName = s3Event.Bucket.Name,
Key = s3Event.Object.Key
});
return(true);
}
catch (Exception e)
{
context.Logger.LogLine($"Error getting object {s3Event.Object.Key} from bucket {s3Event.Bucket.Name}. Make sure they exist and your bucket is in the same region as this function.");
context.Logger.LogLine(e.Message);
context.Logger.LogLine(e.StackTrace);
throw;
}
}
// POST api/values
//[HttpPost("api/upload")]
public async Task <IHttpActionResult> Post()
{
string filename = "";
string json = "";
IndexResponse asyncIndexResponse = null;
string indexName = "1";
if (!Request.Content.IsMimeMultipartContent())
{
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
}
var provider = new MultipartMemoryStreamProvider();
await Request.Content.ReadAsMultipartAsync(provider);
foreach (var file in provider.Contents)
{
filename = file.Headers.ContentDisposition.FileName.Trim('\"');
json = await file.ReadAsStringAsync();
}
Data data = JsonConvert.DeserializeObject <Data>(json);
string extension = System.IO.Path.GetExtension(filename);
string index = filename.Substring(0, filename.Length - extension.Length);
IndexResponse responseString;
for (int i = 0; i < data.summarizedInsights.faces.Count; i++)
{
asyncIndexResponse = await lowLevelClient.IndexAsync <IndexResponse>(index, i.ToString(), PostData.Serializable <Face>(data.summarizedInsights.faces[i]));
}
for (int i = 0; i < data.summarizedInsights.keywords.Count; i++)
{
asyncIndexResponse = await lowLevelClient.IndexAsync <IndexResponse>(index, i.ToString(), PostData.Serializable(data.summarizedInsights.keywords[i]));
}
for (int i = 0; i < data.summarizedInsights.sentiments.Count; i++)
{
asyncIndexResponse = await lowLevelClient.IndexAsync <IndexResponse>(index, i.ToString(), PostData.Serializable(data.summarizedInsights.sentiments[i]));
}
for (int i = 0; i < data.summarizedInsights.audioEffects.Count; i++)
{
asyncIndexResponse = await lowLevelClient.IndexAsync <IndexResponse>(index, i.ToString(), PostData.Serializable(data.summarizedInsights.audioEffects[i]));
}
for (int i = 0; i < data.summarizedInsights.labels.Count; i++)
{
asyncIndexResponse = await lowLevelClient.IndexAsync <IndexResponse>(index, i.ToString(), PostData.Serializable(data.summarizedInsights.labels[i]));
}
for (int i = 0; i < data.summarizedInsights.brands.Count; i++)
{
asyncIndexResponse = await lowLevelClient.IndexAsync <IndexResponse>(index, i.ToString(), PostData.Serializable(data.summarizedInsights.brands[i]));
}
for (int i = 0; i < data.summarizedInsights.emotions.Count; i++)
{
asyncIndexResponse = await lowLevelClient.IndexAsync <IndexResponse>(index, i.ToString(), PostData.Serializable(data.summarizedInsights.emotions[i]));
}
for (int i = 0; i < data.summarizedInsights.topics.Count; i++)
{
asyncIndexResponse = await lowLevelClient.IndexAsync <IndexResponse>(index, i.ToString(), PostData.Serializable((SummarizedInsights.Topic)data.summarizedInsights.topics[i]));
}
//foreach (var obj in data.summarizedInsights.brands)
//{
// var indexResponse2 = nestClient.Index(new IndexRequest<Brand>(obj, "brands"));
//}
//var indexResponse2 = nestClient.Index(data.summarizedInsights.brands[0], new IndexRequest<Brand>((IndexName)"brands"));
//foreach(var obj in data.summarizedInsights.brands)
//{
// asyncIndexResponse = await nestClient.LowLevel.IndexAsync<IndexResponse>("1", PostData.Serializable(obj));
//}
//foreach (var obj in data.summarizedInsights.emotions)
//{
// asyncIndexResponse = await nestClient.LowLevel.IndexAsync<IndexResponse>("1", PostData.Serializable(obj));
//}
//foreach (var obj in data.summarizedInsights.audioEffects)
//{
// asyncIndexResponse = await nestClient.LowLevel.IndexAsync<IndexResponse>("1", PostData.Serializable(obj));
//}
//foreach (var obj in data.summarizedInsights.faces)
//{
// asyncIndexResponse = await nestClient.LowLevel.IndexAsync<IndexResponse>("1", PostData.Serializable(obj));
//}
/*
* asyncIndexResponse = await nestClient.LowLevel.IndexAsync<IndexResponse>("1", PostData.Serializable(data.summarizedInsights.brands));
* asyncIndexResponse = await nestClient.LowLevel.IndexAsync<IndexResponse>("1", PostData.Serializable(data.summarizedInsights.emotions));
* asyncIndexResponse = await nestClient.LowLevel.IndexAsync<IndexResponse>("1", PostData.Serializable(data.summarizedInsights.faces));
* asyncIndexResponse = await nestClient.LowLevel.IndexAsync<IndexResponse>("1", PostData.Serializable(data.summarizedInsights.keywords));
* asyncIndexResponse = await nestClient.LowLevel.IndexAsync<IndexResponse>("1", PostData.Serializable(data.summarizedInsights.labels));
* asyncIndexResponse = await nestClient.LowLevel.IndexAsync<IndexResponse>("1", PostData.Serializable(data.summarizedInsights.name));
* asyncIndexResponse = await nestClient.LowLevel.IndexAsync<IndexResponse>("1", PostData.Serializable(data.summarizedInsights.namedLocations));
* asyncIndexResponse = await nestClient.LowLevel.IndexAsync<IndexResponse>("1", PostData.Serializable(data.summarizedInsights.sentiments));
* asyncIndexResponse = await nestClient.LowLevel.IndexAsync<IndexResponse>("1", PostData.Serializable(data.summarizedInsights.topics));
*/
return(Ok(asyncIndexResponse));
}
