/// <summary>
        /// 批量插入文档
        /// </summary>
        /// <param name="docs"></param>
        /// <returns></returns>
        public bool InsertBatchDocument(IEnumerable <string> docs)
        {
            bool flag = false;

            StringResponse resStr = null;

            try
            {
                resStr = client.Bulk <StringResponse>(PostData.MultiJson(docs));
                var resObj = JObject.Parse(resStr.Body);
                if (!(bool)resObj["errors"])
                {
                    flag = true;
                }
                else
                {
                    LogUtil.LogInfo(logger, resStr.DebugInformation, nodeId);
                }
            }
            catch (Exception ex)
            {
                if (resStr != null)
                {
                    LogUtil.LogInfo(logger, resStr.DebugInformation, nodeId);
                }
                LogUtil.LogError(logger, ex.ToString(), nodeId);
            }

            return(flag);
        }
Esempio n. 2
0
        /// <summary>
        /// Main
        /// </summary>
        /// <param name="args">参数</param>
        static void Main(string[] args)
        {
            var settings = new ConnectionConfiguration(new Uri("http://localhost:9200"))
                           .RequestTimeout(TimeSpan.FromMinutes(2));
            var client = new ElasticLowLevelClient(settings);

            //新增一个人

            /*
             * var person = new Person {
             *  FirstName = "Martign",
             *  LastName = "Laarman"
             * };
             * var indexResponse = client.Index<BytesResponse>("people", "person", "1", PostData.Serializable(person));
             * var responseBytes = indexResponse.Body;
             */

            //批量新增
            var persons = new object[] {
                new { index = new { _index = "people", _type = "person", _id = "2" } },
                new { FirstName = "Russ", LastName = "Cam" },
                new { index = new { _index = "people", _type = "person", _id = "3" } },
                new { FirstName = "Rose", LastName = "Cam" },
                new { index = new { _index = "people", _type = "person", _id = "4" } },
                new { FirstName = "Malilian", LastName = "Menluo" }
            };
            var bulkResponse   = client.Bulk <StringResponse>(PostData.MultiJson(persons));
            var responseStream = bulkResponse.Body;
        }
        /// <summary>
        /// Perform an operation with the specified data.
        /// </summary>
        /// <param name="bulkDataJson">Bulk data in JSON format.</param>
        protected void Bulk(String bulkDataJson)
        {
            Boolean succeeded;
            ElasticsearchResponse <DynamicResponse> response;

            succeeded = false;
            while (!succeeded)
            {
                try
                {
                    response = _client.Bulk <DynamicResponse>(bulkDataJson);
                    CheckResponse(response);
                    succeeded = true;
                }
                catch (Exception)
                {
                    //if (Configuration.Debug)
                    //{
                    //    throw;
                    //}
                    // Maybe Elasticsearch is to busy. Try again after a minute.
                    Thread.Sleep(60000);
                    succeeded = false;
                }
            }
        }
Esempio n. 4
0
        public static void execute(string host, int port, Dictionary <ulong, FileNameAndParentFrn> files, int chunkSize = 10000)
        {
            var settings = new ConnectionConfiguration(new Uri("http://" + host + ":" + port))
                           .RequestTimeout(TimeSpan.FromMinutes(2));
            var lowlevelClient = new ElasticLowLevelClient(settings);

            var index = DateTime.Now.ToString("yyyyMMddHHmmss");
            var type  = "pathes";

            long id = 1;

            foreach (var chank in files.Values.Chunks(chunkSize))
            {
                var json = new List <object>();
                foreach (var file in chank)
                {
                    json.Add(new Index(id, index, type));
                    json.Add(new FileEntry(file.Path, ""));

                    id++;
                }

                var indexResponse = lowlevelClient.Bulk <StreamResponse>(PostData.MultiJson(json));
                using (var responseStream = indexResponse.Body)
                {
                    Console.Write(".");
                }
            }
            Console.WriteLine("");
        }
Esempio n. 5
0
        /*
         * dorobic SSL
         */
        public void SendJsonToElastic(List <string> json)
        {
            PostData body     = PostData.MultiJson(json);
            var      response = client.Bulk <BytesResponse>(indexname, body);

            foreach (string json_line in json)
            {
                Console.WriteLine(json_line);
            }
        }
Esempio n. 6
0
        static void BulkIndexing()
        {
            var peoples = new object[]
            {
                new { index = new { _index = "people", _type = "person", _id = "1" } },
                new { FirstName = "Martijn", LastName = "Laarman" },
                new { index = new { _index = "people", _type = "person", _id = "2" } },
                new { FirstName = "Greg", LastName = "Marzouka" },
                new { index = new { _index = "people", _type = "person", _id = "3" } },
                new { FirstName = "Russ", LastName = "Cam" },
            };

            var    indexResponse  = client.Bulk <StringResponse>(PostData.MultiJson(peoples));
            string responseStream = indexResponse.Body;
        }
        static void Main(string[] args)
        {
            var serviceCollection = new ServiceCollection();

            serviceCollection.AddSingleton(new LoggerFactory()
                                           .AddConsole()
                                           .AddDebug());
            serviceCollection.AddLogging();

            var environment = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT");

            if (string.IsNullOrWhiteSpace(environment))
            {
                environment = "Development";
            }

            var configuration = new ConfigurationBuilder()
                                .SetBasePath(Directory.GetCurrentDirectory())
                                .AddJsonFile("appsettings.json", optional: true)
                                .AddJsonFile($"appsettings.{environment}.json", optional: true)
                                .AddEnvironmentVariables("ASPNETCORE_")
                                .Build();

            serviceCollection.AddSingleton <IConfigurationRoot>(configuration);

            var serviceProvider = serviceCollection.BuildServiceProvider();
            var _configuration  = serviceProvider.GetRequiredService <IConfigurationRoot>();

            var options = _configuration.GetAWSOptions();

            var people = new List <Object>();

            for (int i = 0; i < 100; i++)
            {
                string _id = Guid.NewGuid().ToString();
                people.Add(new { index = new { _index = "people", _type = "person", _id = _id } });
                people.Add(new
                {
                    first_name = "name " + i,
                });
            }

            var    elasticURI     = _configuration.GetSection("WorkerSettings").GetSection("ElasticURI").Value;
            var    settings       = new ConnectionConfiguration(new Uri(elasticURI)).RequestTimeout(TimeSpan.FromMinutes(2));
            var    lowlevelClient = new ElasticLowLevelClient(settings);
            var    indexResponse  = lowlevelClient.Bulk <StringResponse>(PostData.MultiJson(people.ToArray()));
            string responseStream = indexResponse.Body;
        }
 private void SendToElasticsearch(List <string> jsonList)
 {
     try
     {
         var settings = new ConnectionConfiguration(new Uri("http://34.90.149.218:9200")).
                        BasicAuthentication("user", "JVVyX6FCgBWo").
                        RequestTimeout(TimeSpan.FromMinutes(2));
         var lowLevelClient = new ElasticLowLevelClient(settings);
         lowLevelClient.Bulk <StringResponse>(PostData.MultiJson(jsonList));
     }
     catch
     {
         // If logging to elasticsearch failed, it could be logged in a file.
         // But I guess that question could be discussed with lecturer.
     }
 }
Esempio n. 9
0
        static void Main(string[] args)
        {
            var             uri            = "http://10.15.9.113:9200";
            var             nodes          = uri.Split(',', StringSplitOptions.RemoveEmptyEntries).Select(url => new Uri(url));
            IConnectionPool connectionPool = new StaticConnectionPool(nodes);
            var             config         = new ConnectionConfiguration(connectionPool);
            //config.BasicAuthentication("", "");

            IElasticLowLevelClient client = new ElasticLowLevelClient(config);



            var document = new Dictionary <string, object>
            {
                { "source", "wpf" },
                { "@timestamp", DateTime.Now },
                { "level", "Info" },
                { "message", "test.log" }
            };

            document["traceId"] = "testTraceId";

            object documentInfo = new { index = new { _index = "wpf-client" } };
            var    payload      = new List <object> {
                documentInfo, document
            };
            var postData = PostData.MultiJson(payload);

            var result    = client.Bulk <BytesResponse>(postData);
            var exception = result.Success
                ? null
                : result.OriginalException ??
                            new Exception("No error message. Enable Trace logging for more information.");

            if (exception != null)
            {
                Console.WriteLine($"ElasticSearch: Failed to send log messages. status={result.HttpStatusCode}");
            }

            Console.WriteLine("Hello World!");
            Console.ReadLine();
        }
        private string CreateRequest(Exception ex)
        {
            var document = new Dictionary <string, object>
            {
                { "message", ObjectMessage },
                { "exception", ex }
            };


            var payload = new List <object>
            {
                BulkHeader,
                document
            };
            var response = _client.Bulk <BytesResponse>(PostData.MultiJson(payload));


            var request = Encoding.UTF8.GetString(response.RequestBodyInBytes);

            return(request);
        }
Esempio n. 11
0
        static void Main(string[] args)
        {
            Console.WriteLine("Starting Connection with ElasticSearch");

            string index      = args.Length > 0 ?args[0]: "testedotnetcore01";
            int    quantidade = args.Length > 1 ? Convert.ToInt32(args[1]) : 5000000;
            int    lote       = args.Length > 2 ? Convert.ToInt32(args[2]) : 10000;
            int    loteIndex  = 1;

            var settings = new ConnectionConfiguration(new Uri("http://*****:*****@172.16.1.174:9200"))
                           .RequestTimeout(TimeSpan.FromMinutes(2));

            var lowlevelClient = new ElasticLowLevelClient(settings);

            var people = new List <object>();

            for (int i = 1; i <= quantidade; i++)
            {
                Console.Write($"\rProcesando o item: {i}");
                people.Add(new { index = new { _index = index, _type = "person", _id = i.ToString() } });
                people.Add(new { FirstName = $"peaple{i}", LastName = $"name{i}" });

                if (i % lote == 0)
                {
                    Console.WriteLine($"\rEnviando o lote: {loteIndex}. Quantidade de itens: {people.Count}");
                    try
                    {
                        var    ndexResponse   = lowlevelClient.Bulk <StringResponse>(PostData.MultiJson(people.ToArray()));
                        string responseStream = ndexResponse.Body;
                    }
                    catch (Exception ex)
                    {
                        Console.WriteLine($"\rErro ao enviar lote. {ex.Message}");
                    }

                    people.Clear();
                    loteIndex++;
                }
            }
        }
Esempio n. 12
0
        /// <summary>
        /// Saves log messages from the cache.
        /// </summary>
        /// <param name="messages">a list with log messages</param>
        protected override void Save(List <LogMessage> messages)
        {
            if (messages == null || messages.Count == 0)
            {
                return;
            }

            if (_client == null)
            {
                throw new InvalidStateException("elasticsearch_logger", "NOT_OPENED", "ElasticSearchLogger is not opened");
            }

            lock (_lock)
            {
                CreateIndex("elasticsearch_logger", false).Wait();

                var bulk = new List <string>();
                foreach (var message in messages)
                {
                    bulk.Add(JsonConverter.ToJson(new { index = new { _index = _currentIndexName, _type = "log_message", _id = IdGenerator.NextLong() } }));
                    bulk.Add(JsonConverter.ToJson(message));
                }

                try
                {
                    var response = _client.Bulk <StringResponse>(PostData.MultiJson(bulk));
                    if (!response.Success)
                    {
                        throw new InvocationException("elasticsearch_logger", "REQUEST_FAILED", response.Body);
                    }
                }
                catch
                {
                    // Do nothing if elastic search client was not enable to process bulk of messages
                    _errorConsoleLogger.Error(null, "Failed to bulk messages with Elastic Search Logger.");
                }
            }
        }
Esempio n. 13
0
        public static void InsertIntoES(string csv_filename, ElasticLowLevelClient lowLevelClient, bool isAsync, int bulkLimit, int shardCount, int replicaCount)
        {
            var idelResponse = lowLevelClient.IndicesDelete <BytesResponse>("people");

            var icrResponse = lowLevelClient.IndicesCreate <BytesResponse>("people", PostData.Serializable(
                                                                               new
            {
                settings = new
                {
                    number_of_shards   = shardCount,
                    number_of_replicas = replicaCount
                },
                mappings = new
                {
                    _doc = new
                    {
                        properties = new
                        {
                            FirstName      = new { type = "text" },
                            LastName       = new { type = "text" },
                            UserName       = new { type = "text" },
                            SSC_Grade      = new { type = "text" },
                            HSC_Grade      = new { type = "text" },
                            Bachelor_Grade = new { type = "text" },
                            Age            = new { type = "integer" },
                            Gender         = new { type = "text" },
                            Email          = new { type = "text" },
                            DateOfBirth    = new { type = "date" },
                            Streent        = new { type = "text" },
                            Suite          = new { type = "text" },
                            City           = new { type = "text" },
                            ZipCode        = new { type = "text" },
                        }
                    }
                }
            }
                                                                               ));

            var people = new List <object>
            {
                new { index = new { _index = "people", _type = "_doc" } }
            };


            int counter = 0;
            int id      = 0;


            using (StreamReader reader = new StreamReader(csv_filename))
            {
                reader.ReadLine();
                while (true)
                {
                    string line = reader.ReadLine();
                    if (line == null)
                    {
                        break;
                    }

                    var values = line.Split(",");

                    if (id == 0)
                    {
                        people = new List <object>
                        {
                        };
                    }

                    if (counter >= bulkLimit)
                    {
                        if (isAsync)
                        {
                            lowLevelClient.BulkAsync <StringResponse>(PostData.MultiJson(people));
                        }
                        else
                        {
                            lowLevelClient.Bulk <StringResponse>(PostData.MultiJson(people));
                        }

                        counter = 0;
                        people  = new List <object>
                        {
                        };
                    }
                    else
                    {
                        people.Add(new { index = new { _index = "people", _type = "_doc", _id = $"{id}" } });

                        people.Add(
                            new
                        {
                            FirstName      = values[0],
                            LastName       = values[1],
                            UserName       = values[2],
                            SSC_Grade      = values[3],
                            HSC_Grade      = values[4],
                            Bachelor_Grade = values[5],
                            Age            = Int32.Parse(values[6]),
                            Gender         = values[7],
                            Email          = values[8],
                            DateOfBirth    = DateTime.Parse(values[9]),
                            Street         = values[10],
                            Suite          = values[11],
                            City           = values[12],
                            ZipCode        = values[13],
                        });
                        counter++;
                        id++;
                    }


                    Console.WriteLine(id);
                }


                if (isAsync)
                {
                    lowLevelClient.BulkAsync <StringResponse>(PostData.MultiJson(people));
                }
                else
                {
                    lowLevelClient.Bulk <StringResponse>(PostData.MultiJson(people));
                }
            }
        }