public static ElasticClient EsClient() { ESClusterConfiguration esClusterConfig = new ESClusterConfiguration(); ConnectionSettings connectionSettings; ElasticClient elasticClient; StaticConnectionPool connectionPool; string index = esClusterConfig.ESConfigSection.IndexName; //Multiple node for fail over (cluster addresses) var nodes = new List <Uri>(); foreach (NodeElement node in esClusterConfig.ESConfigSection.Nodes) { nodes.Add(new Uri(node.Uri)); } connectionPool = new StaticConnectionPool(nodes.ToArray()); //Connection string for Elasticsearch connectionSettings = new ConnectionSettings(connectionPool) .DefaultIndex(index) .MaximumRetries(2) .MaxRetryTimeout(TimeSpan.FromSeconds(150)); elasticClient = new ElasticClient(connectionSettings); //connectionSettings = new ConnectionSettings(connectionPool); elasticClient = new ElasticClient(connectionSettings); // create the index if it doesn't exist if (!elasticClient.IndexExists(index).Exists) { elasticClient.CreateIndex(index); } return(elasticClient); }
static void Main(string[] args) { ESClusterConfiguration esClusterConfig = new ESClusterConfiguration(); try { #region This is for reading Salaries & Interests from a Sample Json IEnumerable <SampleData> sampleModel; List <SampleData> listSampleModel; string path = ConfigurationManager.AppSettings["SampleJsonPath"]; using (StreamReader r = new StreamReader(path)) { JObject obj = JObject.Parse(r.ReadToEnd()); var list = (JArray)obj.SelectToken("results"); sampleModel = from item in (JArray)obj["results"] select new SampleData { Salary = item["Salary"].Value <string>(), Interests = item["Interests"].Value <string>() }; } listSampleModel = sampleModel.ToList(); #endregion //Call randomuser.me API for fetching random users HttpClient client = new HttpClient(); client.BaseAddress = new Uri(URL); // Add an Accept header for JSON format. client.DefaultRequestHeaders.Accept.Add( new MediaTypeWithQualityHeaderValue("application/json")); DataTable tbl = new DataTable("PersonTable"); tbl.Columns.Add("ssn", typeof(String)); tbl.Columns.Add("gender", typeof(String)); tbl.Columns.Add("firstname", typeof(String)); tbl.Columns.Add("lastname", typeof(String)); tbl.Columns.Add("email", typeof(String)); tbl.Columns.Add("dob", typeof(String)); tbl.Columns.Add("cell", typeof(String)); tbl.Columns.Add("salary", typeof(decimal)); tbl.Columns.Add("interests", typeof(String)); tbl.Columns.Add("json_data", typeof(String)); tbl.Columns.Add("street", typeof(String)); tbl.Columns.Add("state", typeof(String)); tbl.Columns.Add("city", typeof(String)); tbl.Columns.Add("zip", typeof(String)); //tbl.Columns.Add("description", typeof(String)); DataRow row = null; if (!int.TryParse(ConfigurationManager.AppSettings["DataImportLoopCounter"], out int loopCounter)) { loopCounter = 1; } for (int i = 0; i < loopCounter; i++) { try { if (i > 0) { //Taking some pause as API can't handle too many requests Thread.Sleep(10000); } Console.WriteLine("Counter : " + i); HttpResponseMessage response = client.GetAsync(urlParameters).Result; // Blocking call! if (response.IsSuccessStatusCode) { string json = response.Content.ReadAsStringAsync().Result; PersonResponse personResp = JsonConvert.DeserializeObject <PersonResponse>(json); //Inject Salary & Interest for (int j = 0; j < RecordCountInsert; j++) { personResp.results[j].Salary = Convert.ToDecimal(listSampleModel[j].Salary); personResp.results[j].Interests = listSampleModel[j].Interests; } //Serialize to json again json = JsonConvert.SerializeObject(personResp); JObject obj = JObject.Parse(json); var list = (JArray)obj.SelectToken("results"); var listPerson = (JArray)obj.SelectToken("results"); for (int k = 0; k < RecordCountInsert; k++) { row = tbl.NewRow(); row["ssn"] = personResp.results[k].id.value; row["gender"] = personResp.results[k].gender; row["firstname"] = personResp.results[k].name.first; row["lastname"] = personResp.results[k].name.last; row["email"] = personResp.results[k].email; row["dob"] = personResp.results[k].dob; row["cell"] = personResp.results[k].cell; row["salary"] = personResp.results[k].Salary; row["interests"] = personResp.results[k].Interests; row["street"] = personResp.results[k].location.street; row["state"] = personResp.results[k].location.state; row["city"] = personResp.results[k].location.city; row["zip"] = Convert.ToString(personResp.results[k].location.postcode); //row["description"] = "Dummy text"; // Praveer can use this field //Creating PersonRecord object as we want to store data PersonRecord ps = new PersonRecord { IdType = personResp.results[k].id.name, Id = personResp.results[k].id.value, Gender = personResp.results[k].gender, Title = personResp.results[k].name.title, FirstName = personResp.results[k].name.first, LastName = personResp.results[k].name.last, Email = personResp.results[k].email, Dob = personResp.results[k].dob, Cell = personResp.results[k].cell, Salary = personResp.results[k].Salary, Interests = personResp.results[k].Interests, Street = personResp.results[k].location.street, City = personResp.results[k].location.city, State = personResp.results[k].location.state, PostCode = personResp.results[k].location.postcode }; row["json_data"] = JsonConvert.SerializeObject(ps); tbl.Rows.Add(row); } Console.WriteLine("Counter : " + i + " DataTable Rows Count : " + tbl.Rows.Count); } else { Console.WriteLine("Bad response received"); } Console.WriteLine("DataTable Rows Count Before Insert : " + tbl.Rows.Count); CopyDataToSqlServer(tbl, "Person"); InsertJsonToElastic(tbl); tbl.Clear(); } catch (Exception ex) //Catch exception in for loop and then continue { Console.WriteLine("Contine after Exception : " + ex.Message); tbl.Clear(); continue; } } } catch (Exception e) //Catch unhandled exception and stops { Console.WriteLine("Exception : " + e.Message); Console.WriteLine("Exception StackTrace: " + e.StackTrace); } Console.WriteLine("Insert Done"); Console.ReadLine(); }