public static string InsertIntoElastic(IEnumerable <Package> packages)
{
//TODO: make these config values
string url = ElasticURL.ToString();
if (packages != null && packages.Count() > 0)
{
StringBuilder sb = new StringBuilder();
int numLines = 0;
string eh = string.Empty;
foreach (Package p in packages)
{
try
{
DateTime start = DateTime.Now;
sb.AppendLine(string.Format("{{ \"index\": {{ \"_id\":\"{0}\" }} }}", p.NdcPackageCode));
sb.AppendLine(Newtonsoft.Json.JsonConvert.SerializeObject(p, Newtonsoft.Json.Formatting.None, new Newtonsoft.Json.JsonSerializerSettings {
NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore
}));
numLines++;
if (numLines == 10000)
{
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(string.Format("{0}data/package/_bulk", url));
req.ContentType = "application/json";
req.Method = "PUT";
eh = req.GetResponseString(sb.ToString());
Console.WriteLine(String.Format("sent {0} records in {1}", numLines, DateTime.Now - start));
sb.Clear();
numLines = 0;
}
}
catch (Exception e)
{
return(e.Message);
}
}
if (!String.IsNullOrEmpty(eh))
{
HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(string.Format("{0}data/package/_bulk", url));
req.ContentType = "application/json";
req.Method = "PUT";
eh = req.GetResponseString(sb.ToString());
}
return(eh);
}
return("");
}
public static void InsertIntoElastic(string documentType, string[] jsonFiles)
{
using (var client = new WebClient())
{
foreach (var file in jsonFiles)
{
JObject j = JObject.Parse(File.ReadAllText(file));
string id = j["id"].ToString();
string url = ElasticURL.ToString();
string elasticUrl = string.Format("{0}data/{1}/", url, documentType);
elasticUrl = elasticUrl + id;
byte[] data = File.ReadAllBytes(file);
client.UploadData(elasticUrl, "PUT", data);
Console.WriteLine(file + " was successfully added to ElasticSearch");
}
}
}