/// <summary>
/// 批量插入文档
/// </summary>
/// <param name="docs"></param>
/// <returns></returns>
public bool InsertBatchDocument(IEnumerable <string> docs)
{
bool flag = false;
StringResponse resStr = null;
try
{
resStr = client.Bulk <StringResponse>(PostData.MultiJson(docs));
var resObj = JObject.Parse(resStr.Body);
if (!(bool)resObj["errors"])
{
flag = true;
}
else
{
LogUtil.LogInfo(logger, resStr.DebugInformation, nodeId);
}
}
catch (Exception ex)
{
if (resStr != null)
{
LogUtil.LogInfo(logger, resStr.DebugInformation, nodeId);
}
LogUtil.LogError(logger, ex.ToString(), nodeId);
}
return(flag);
}
/// <summary>
/// Main
/// </summary>
/// <param name="args">参数</param>
static void Main(string[] args)
{
var settings = new ConnectionConfiguration(new Uri("http://localhost:9200"))
.RequestTimeout(TimeSpan.FromMinutes(2));
var client = new ElasticLowLevelClient(settings);
//新增一个人
/*
* var person = new Person {
* FirstName = "Martign",
* LastName = "Laarman"
* };
* var indexResponse = client.Index<BytesResponse>("people", "person", "1", PostData.Serializable(person));
* var responseBytes = indexResponse.Body;
*/
//批量新增
var persons = new object[] {
new { index = new { _index = "people", _type = "person", _id = "2" } },
new { FirstName = "Russ", LastName = "Cam" },
new { index = new { _index = "people", _type = "person", _id = "3" } },
new { FirstName = "Rose", LastName = "Cam" },
new { index = new { _index = "people", _type = "person", _id = "4" } },
new { FirstName = "Malilian", LastName = "Menluo" }
};
var bulkResponse = client.Bulk <StringResponse>(PostData.MultiJson(persons));
var responseStream = bulkResponse.Body;
}
/// <summary>
/// Perform an operation with the specified data.
/// </summary>
/// <param name="bulkDataJson">Bulk data in JSON format.</param>
protected void Bulk(String bulkDataJson)
{
Boolean succeeded;
ElasticsearchResponse <DynamicResponse> response;
succeeded = false;
while (!succeeded)
{
try
{
response = _client.Bulk <DynamicResponse>(bulkDataJson);
CheckResponse(response);
succeeded = true;
}
catch (Exception)
{
//if (Configuration.Debug)
//{
// throw;
//}
// Maybe Elasticsearch is to busy. Try again after a minute.
Thread.Sleep(60000);
succeeded = false;
}
}
}
public static void execute(string host, int port, Dictionary <ulong, FileNameAndParentFrn> files, int chunkSize = 10000)
{
var settings = new ConnectionConfiguration(new Uri("http://" + host + ":" + port))
.RequestTimeout(TimeSpan.FromMinutes(2));
var lowlevelClient = new ElasticLowLevelClient(settings);
var index = DateTime.Now.ToString("yyyyMMddHHmmss");
var type = "pathes";
long id = 1;
foreach (var chank in files.Values.Chunks(chunkSize))
{
var json = new List <object>();
foreach (var file in chank)
{
json.Add(new Index(id, index, type));
json.Add(new FileEntry(file.Path, ""));
id++;
}
var indexResponse = lowlevelClient.Bulk <StreamResponse>(PostData.MultiJson(json));
using (var responseStream = indexResponse.Body)
{
Console.Write(".");
}
}
Console.WriteLine("");
}
/*
* dorobic SSL
*/
public void SendJsonToElastic(List <string> json)
{
PostData body = PostData.MultiJson(json);
var response = client.Bulk <BytesResponse>(indexname, body);
foreach (string json_line in json)
{
Console.WriteLine(json_line);
}
}
static void BulkIndexing()
{
var peoples = new object[]
{
new { index = new { _index = "people", _type = "person", _id = "1" } },
new { FirstName = "Martijn", LastName = "Laarman" },
new { index = new { _index = "people", _type = "person", _id = "2" } },
new { FirstName = "Greg", LastName = "Marzouka" },
new { index = new { _index = "people", _type = "person", _id = "3" } },
new { FirstName = "Russ", LastName = "Cam" },
};
var indexResponse = client.Bulk <StringResponse>(PostData.MultiJson(peoples));
string responseStream = indexResponse.Body;
}
static void Main(string[] args)
{
var serviceCollection = new ServiceCollection();
serviceCollection.AddSingleton(new LoggerFactory()
.AddConsole()
.AddDebug());
serviceCollection.AddLogging();
var environment = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT");
if (string.IsNullOrWhiteSpace(environment))
{
environment = "Development";
}
var configuration = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", optional: true)
.AddJsonFile($"appsettings.{environment}.json", optional: true)
.AddEnvironmentVariables("ASPNETCORE_")
.Build();
serviceCollection.AddSingleton <IConfigurationRoot>(configuration);
var serviceProvider = serviceCollection.BuildServiceProvider();
var _configuration = serviceProvider.GetRequiredService <IConfigurationRoot>();
var options = _configuration.GetAWSOptions();
var people = new List <Object>();
for (int i = 0; i < 100; i++)
{
string _id = Guid.NewGuid().ToString();
people.Add(new { index = new { _index = "people", _type = "person", _id = _id } });
people.Add(new
{
first_name = "name " + i,
});
}
var elasticURI = _configuration.GetSection("WorkerSettings").GetSection("ElasticURI").Value;
var settings = new ConnectionConfiguration(new Uri(elasticURI)).RequestTimeout(TimeSpan.FromMinutes(2));
var lowlevelClient = new ElasticLowLevelClient(settings);
var indexResponse = lowlevelClient.Bulk <StringResponse>(PostData.MultiJson(people.ToArray()));
string responseStream = indexResponse.Body;
}
private void SendToElasticsearch(List <string> jsonList)
{
try
{
var settings = new ConnectionConfiguration(new Uri("http://34.90.149.218:9200")).
BasicAuthentication("user", "JVVyX6FCgBWo").
RequestTimeout(TimeSpan.FromMinutes(2));
var lowLevelClient = new ElasticLowLevelClient(settings);
lowLevelClient.Bulk <StringResponse>(PostData.MultiJson(jsonList));
}
catch
{
// If logging to elasticsearch failed, it could be logged in a file.
// But I guess that question could be discussed with lecturer.
}
}
static void Main(string[] args)
{
var uri = "http://10.15.9.113:9200";
var nodes = uri.Split(',', StringSplitOptions.RemoveEmptyEntries).Select(url => new Uri(url));
IConnectionPool connectionPool = new StaticConnectionPool(nodes);
var config = new ConnectionConfiguration(connectionPool);
//config.BasicAuthentication("", "");
IElasticLowLevelClient client = new ElasticLowLevelClient(config);
var document = new Dictionary <string, object>
{
{ "source", "wpf" },
{ "@timestamp", DateTime.Now },
{ "level", "Info" },
{ "message", "test.log" }
};
document["traceId"] = "testTraceId";
object documentInfo = new { index = new { _index = "wpf-client" } };
var payload = new List <object> {
documentInfo, document
};
var postData = PostData.MultiJson(payload);
var result = client.Bulk <BytesResponse>(postData);
var exception = result.Success
? null
: result.OriginalException ??
new Exception("No error message. Enable Trace logging for more information.");
if (exception != null)
{
Console.WriteLine($"ElasticSearch: Failed to send log messages. status={result.HttpStatusCode}");
}
Console.WriteLine("Hello World!");
Console.ReadLine();
}
private string CreateRequest(Exception ex)
{
var document = new Dictionary <string, object>
{
{ "message", ObjectMessage },
{ "exception", ex }
};
var payload = new List <object>
{
BulkHeader,
document
};
var response = _client.Bulk <BytesResponse>(PostData.MultiJson(payload));
var request = Encoding.UTF8.GetString(response.RequestBodyInBytes);
return(request);
}
static void Main(string[] args)
{
Console.WriteLine("Starting Connection with ElasticSearch");
string index = args.Length > 0 ?args[0]: "testedotnetcore01";
int quantidade = args.Length > 1 ? Convert.ToInt32(args[1]) : 5000000;
int lote = args.Length > 2 ? Convert.ToInt32(args[2]) : 10000;
int loteIndex = 1;
var settings = new ConnectionConfiguration(new Uri("http://*****:*****@172.16.1.174:9200"))
.RequestTimeout(TimeSpan.FromMinutes(2));
var lowlevelClient = new ElasticLowLevelClient(settings);
var people = new List <object>();
for (int i = 1; i <= quantidade; i++)
{
Console.Write($"\rProcesando o item: {i}");
people.Add(new { index = new { _index = index, _type = "person", _id = i.ToString() } });
people.Add(new { FirstName = $"peaple{i}", LastName = $"name{i}" });
if (i % lote == 0)
{
Console.WriteLine($"\rEnviando o lote: {loteIndex}. Quantidade de itens: {people.Count}");
try
{
var ndexResponse = lowlevelClient.Bulk <StringResponse>(PostData.MultiJson(people.ToArray()));
string responseStream = ndexResponse.Body;
}
catch (Exception ex)
{
Console.WriteLine($"\rErro ao enviar lote. {ex.Message}");
}
people.Clear();
loteIndex++;
}
}
}
/// <summary>
/// Saves log messages from the cache.
/// </summary>
/// <param name="messages">a list with log messages</param>
protected override void Save(List <LogMessage> messages)
{
if (messages == null || messages.Count == 0)
{
return;
}
if (_client == null)
{
throw new InvalidStateException("elasticsearch_logger", "NOT_OPENED", "ElasticSearchLogger is not opened");
}
lock (_lock)
{
CreateIndex("elasticsearch_logger", false).Wait();
var bulk = new List <string>();
foreach (var message in messages)
{
bulk.Add(JsonConverter.ToJson(new { index = new { _index = _currentIndexName, _type = "log_message", _id = IdGenerator.NextLong() } }));
bulk.Add(JsonConverter.ToJson(message));
}
try
{
var response = _client.Bulk <StringResponse>(PostData.MultiJson(bulk));
if (!response.Success)
{
throw new InvocationException("elasticsearch_logger", "REQUEST_FAILED", response.Body);
}
}
catch
{
// Do nothing if elastic search client was not enable to process bulk of messages
_errorConsoleLogger.Error(null, "Failed to bulk messages with Elastic Search Logger.");
}
}
}
public static void InsertIntoES(string csv_filename, ElasticLowLevelClient lowLevelClient, bool isAsync, int bulkLimit, int shardCount, int replicaCount)
{
var idelResponse = lowLevelClient.IndicesDelete <BytesResponse>("people");
var icrResponse = lowLevelClient.IndicesCreate <BytesResponse>("people", PostData.Serializable(
new
{
settings = new
{
number_of_shards = shardCount,
number_of_replicas = replicaCount
},
mappings = new
{
_doc = new
{
properties = new
{
FirstName = new { type = "text" },
LastName = new { type = "text" },
UserName = new { type = "text" },
SSC_Grade = new { type = "text" },
HSC_Grade = new { type = "text" },
Bachelor_Grade = new { type = "text" },
Age = new { type = "integer" },
Gender = new { type = "text" },
Email = new { type = "text" },
DateOfBirth = new { type = "date" },
Streent = new { type = "text" },
Suite = new { type = "text" },
City = new { type = "text" },
ZipCode = new { type = "text" },
}
}
}
}
));
var people = new List <object>
{
new { index = new { _index = "people", _type = "_doc" } }
};
int counter = 0;
int id = 0;
using (StreamReader reader = new StreamReader(csv_filename))
{
reader.ReadLine();
while (true)
{
string line = reader.ReadLine();
if (line == null)
{
break;
}
var values = line.Split(",");
if (id == 0)
{
people = new List <object>
{
};
}
if (counter >= bulkLimit)
{
if (isAsync)
{
lowLevelClient.BulkAsync <StringResponse>(PostData.MultiJson(people));
}
else
{
lowLevelClient.Bulk <StringResponse>(PostData.MultiJson(people));
}
counter = 0;
people = new List <object>
{
};
}
else
{
people.Add(new { index = new { _index = "people", _type = "_doc", _id = $"{id}" } });
people.Add(
new
{
FirstName = values[0],
LastName = values[1],
UserName = values[2],
SSC_Grade = values[3],
HSC_Grade = values[4],
Bachelor_Grade = values[5],
Age = Int32.Parse(values[6]),
Gender = values[7],
Email = values[8],
DateOfBirth = DateTime.Parse(values[9]),
Street = values[10],
Suite = values[11],
City = values[12],
ZipCode = values[13],
});
counter++;
id++;
}
Console.WriteLine(id);
}
if (isAsync)
{
lowLevelClient.BulkAsync <StringResponse>(PostData.MultiJson(people));
}
else
{
lowLevelClient.Bulk <StringResponse>(PostData.MultiJson(people));
}
}
}
