public List <JsonEvent> ProcJson(string InputObject, string ParentObjectKey, bool DoInputString)
{
Stopwatch sw = new Stopwatch();
//Stopwatch sw2 = new Stopwatch();
sw.Start();
List <JsonEvent> jsonEvents = new List <JsonEvent>();
//JsonEvent je = new JsonEvent();
//SortedList<int, string> tmpFieldOrder = new SortedList<int, string>();
//SortedList<int, JsonField> tmpFields = new SortedList<int, JsonField>();
string jsonStr = "";
if (DoInputString)
{
jsonStr = InputObject;
}
else
{
using (FileStream fs = new FileStream(InputObject, FileMode.Open, FileAccess.ReadWrite))
{
StreamReader sr = new StreamReader(fs);
jsonStr = sr.ReadToEnd();
sr.Close();
fs.Close();
}
}
JObject jpobj = (JObject)JsonConvert.DeserializeObject(jsonStr);
JToken jptk = jpobj[ParentObjectKey];
if (jptk.Type.ToString() != "Array")
{
Util.DateLog(@"ERROR: Parent object is not an array!");
throw new Exception(@"Parent object is not an array!");
}
int totChilds = jptk.Count();
Util.DateLog(String.Format(@"Utilizing parent object: {0}", ParentObjectKey));
Util.DateLog(String.Format(@"Found {0} child objects to process", totChilds));
//foreach (JToken jtk in jptk.Values())
//foreach (JToken jtk in jptk) // Good
foreach (JObject jobj in jptk)
{
//sw2.Start();
JsonEvent je = new JsonEvent();
//SortedList<int, string> tmpFieldOrder = new SortedList<int, string>();
SortedList <int, JsonField> tmpFields = new SortedList <int, JsonField>();
int tmpFieldCounter = 1;
foreach (JProperty jp in jobj.Properties())
{
JsonField jf = new JsonField(tmpFieldCounter, jp.Name, jp.Value.Type.ToString(), null);
switch (jp.Value.Type.ToString())
{
case "String":
jf.Value = (string)((JValue)jp.Value.ToString());
break;
case "Null":
jf.Value = "null";
break;
case "Object":
// Method 1 - leave as string
//jf.Value = new object();
//jf.Value = (object)jp.Value;
// Method 2 - Second order conversion
jf.Value = (string)@"object";
JObject tmpChildJson = (JObject)JsonConvert.DeserializeObject(jp.Value.ToString());
this.ProcJsonChild2(tmpChildJson, jp.Name, ref tmpFieldCounter, ref tmpFields);
break;
case "Int":
jf.Value = new int();
jf.Value = (int)jp.Value;
break;
default:
jf.Value = new object();
jf.Value = (object)jp.Value;
break;
}
tmpFields.Add(tmpFieldCounter, jf);
//tmpFieldOrder.Add(tmpFieldCounter, jp.Name);
tmpFieldCounter++;
}
je.ImportFields(tmpFields);
je.GetLogOutput2(false);
jsonEvents.Add(je);
//sw2.Stop();
//Util.DateLog(String.Format(@"Conversion time: {0}", sw2.Elapsed));
//sw2.Reset();
}
sw.Stop();
Util.DateLog(String.Format(@"Total conversion time: {0}", sw.Elapsed));
double dEventRate = (double)(totChilds / sw.Elapsed.TotalSeconds);
//Util.DateLog(String.Format(@"Conversion rate: {1}{0:F2} logs/sec{2}", dEventRate, "\x1B[92m", "\x1B[0m"));
Util.DateLog(String.Format(@"Conversion rate: {0:F2} logs/sec", dEventRate));
return(jsonEvents);
}
public List <JsonEvent> ProcJson5(string InputObject, string ParentObjectKey, bool DoInputString)
{
Stopwatch sw = new Stopwatch();
//Stopwatch sw2 = new Stopwatch();
sw.Start();
List <JsonEvent> jsonEvents = new List <JsonEvent>();
//JsonEvent je = new JsonEvent();
//SortedList<int, string> tmpFieldOrder = new SortedList<int, string>();
//SortedList<int, JsonField> tmpFields = new SortedList<int, JsonField>();
// Set up temporary hash table to test field ordering
Hashtable hPrefFieldOrder = new Hashtable();
/*hPrefFieldOrder.Add(@"eventTime", 1);
* hPrefFieldOrder.Add(@"eventType", 2);
* hPrefFieldOrder.Add(@"threatScore", 3);
* hPrefFieldOrder.Add(@"longDescription", -1);*/
string jsFieldOrder = Properties.Settings.Default.fieldSettings;
JObject joFieldOrder = (JObject)JsonConvert.DeserializeObject(jsFieldOrder);
foreach (var jf in joFieldOrder)
{
hPrefFieldOrder.Add(jf.Key, (int)jf.Value);
}
string jsonStr = "";
if (DoInputString)
{
jsonStr = InputObject;
}
else
{
using (FileStream fs = new FileStream(InputObject, FileMode.Open, FileAccess.ReadWrite))
{
StreamReader sr = new StreamReader(fs);
jsonStr = sr.ReadToEnd();
sr.Close();
fs.Close();
}
}
JObject jpobj = (JObject)JsonConvert.DeserializeObject(jsonStr);
JToken jptk = jpobj[ParentObjectKey];
if (jptk.Type.ToString() != "Array")
{
Util.DateLog(@"ERROR: Parent object is not an array!");
throw new Exception(@"Parent object is not an array!");
}
int totChilds = jptk.Count();
Util.DateLog(String.Format(@"Utilizing parent object: {0}", ParentObjectKey));
Util.DateLog(String.Format(@"Found {0} child objects to process", totChilds));
//foreach (JToken jtk in jptk.Values())
//foreach (JToken jtk in jptk) // Good
foreach (JObject jobj in jptk)
{
//sw2.Start();
JsonEvent je = new JsonEvent();
//SortedList<int, string> tmpFieldOrder = new SortedList<int, string>();
SortedList <int, JsonField> tmpFields = new SortedList <int, JsonField>();
int tmpFieldCounter = 1;
foreach (JProperty jp in jobj.Properties())
{
JsonField jf = new JsonField(tmpFieldCounter, jp.Name, jp.Value.Type.ToString(), null);
switch (jp.Value.Type.ToString())
{
case "String":
jf.Value = (string)((JValue)jp.Value.ToString());
break;
case "Null":
jf.Value = "null";
break;
case "Object":
// Method 1 - leave as string
//jf.Value = new object();
//jf.Value = (object)jp.Value;
// Method 2 - Second order conversion
jf.Value = (string)@"object";
JObject tmpChildJson = (JObject)JsonConvert.DeserializeObject(jp.Value.ToString());
this.ProcJsonChild2(tmpChildJson, jp.Name, ref tmpFieldCounter, ref tmpFields);
break;
case "Int":
jf.Value = new int();
jf.Value = (int)jp.Value;
break;
case "Integer":
jf.Value = new Int64();
jf.Value = (Int64)jp.Value;
break;
case "Array":
string aTmpStr = @"";
foreach (JToken jt in jp.Children())
{
//aTmpStr += jt.ToString();
//aTmpStr += jt.Value<string>().ToString();
string[] aTmp = jt.ToObject <string[]>();
foreach (string s in aTmp)
{
aTmpStr += s + ",";
}
}
aTmpStr = aTmpStr.TrimEnd(",".ToCharArray());
jf.TypeName = "String";
jf.Value = (string)aTmpStr;
break;
default:
jf.Value = new object();
jf.Value = (object)jp.Value;
break;
}
tmpFields.Add(tmpFieldCounter, jf);
//tmpFieldOrder.Add(tmpFieldCounter, jp.Name);
tmpFieldCounter++;
}
// Perform field ordering here
// ============ METHOD 2
Dictionary <string, int> dTmpFieldNames = new Dictionary <string, int>();
Dictionary <string, string> dTmpFieldValues = new Dictionary <string, string>();
Dictionary <string, string> dTmpFieldTypes = new Dictionary <string, string>();
foreach (KeyValuePair <int, JsonField> kvpjf in tmpFields)
{
dTmpFieldNames.Add(kvpjf.Value.Name, kvpjf.Key);
dTmpFieldValues.Add(kvpjf.Value.Name, kvpjf.Value.Value.ToString());
dTmpFieldTypes.Add(kvpjf.Value.Name, kvpjf.Value.TypeName);
}
string tmpLogOut = @"";
//var orderedFields = dTmpFieldNames.Where(f => hPrefFieldOrder.ContainsKey(f.Key) && ((int)hPrefFieldOrder[f.Key] != -1));
var orderedFields = dTmpFieldNames.Where(f => hPrefFieldOrder.ContainsKey(f.Key) && ((int)hPrefFieldOrder[f.Key] != -1)).OrderBy(f2 => (int)hPrefFieldOrder[f2.Key]);
foreach (var of in orderedFields)
{
tmpLogOut += String.Format(@"{0}={1}|", of.Key, dTmpFieldValues[of.Key]);
}
var unorderedFields = dTmpFieldNames.Where(f => hPrefFieldOrder.ContainsKey(f.Key) == false && dTmpFieldTypes[f.Key] != "Array" && dTmpFieldTypes[f.Key] != "Object");
foreach (var uf in unorderedFields)
{
tmpLogOut += String.Format(@"{0}={1}|", uf.Key, dTmpFieldValues[uf.Key]);
}
Regex r = new Regex(@"eventTime=(\d{13})", RegexOptions.None);
if (r.IsMatch(tmpLogOut))
{
Match m = r.Match(tmpLogOut);
string matchStr = m.Groups[0].Value;
string repStr = m.Groups[1].Value;
string newDate = String.Format(@"eventTime={0}", Util.GetEpochToDate(repStr));
tmpLogOut = tmpLogOut.Replace(matchStr, newDate);
}
tmpLogOut = tmpLogOut.TrimEnd("|".ToCharArray());
je.SetLogOutput(tmpLogOut);
//Console.WriteLine(tmpLogOut);
jsonEvents.Add(je);
//sw2.Stop();
//Util.DateLog(String.Format(@"Conversion time: {0}", sw2.Elapsed));
//sw2.Reset();
}
sw.Stop();
Util.DateLog(String.Format(@"Total conversion time: {0}", sw.Elapsed));
double dEventRate = (double)(totChilds / sw.Elapsed.TotalSeconds);
//Util.DateLog(String.Format(@"Conversion rate: {1}{0:F2} logs/sec{2}", dEventRate, "\x1B[92m", "\x1B[0m"));
Util.DateLog(String.Format(@"Conversion rate: {0:F2} logs/sec", dEventRate));
return(jsonEvents);
}
public JsonEvent ProcJson(string InputFile)
{
Stopwatch sw = new Stopwatch();
sw.Start();
JsonEvent je = new JsonEvent();
SortedList <int, string> tmpFieldOrder = new SortedList <int, string>();
SortedList <int, JsonField> tmpFields = new SortedList <int, JsonField>();
string jsonStr = "";
using (FileStream fs = new FileStream(InputFile, FileMode.Open, FileAccess.ReadWrite))
{
StreamReader sr = new StreamReader(fs);
jsonStr = sr.ReadToEnd();
sr.Close();
fs.Close();
}
JObject jobj = (JObject)JsonConvert.DeserializeObject(jsonStr);
int tmpFieldCounter = 1;
// ========== Property Enumeration/Processing, rev 1: DEPRECATED, using rev 2 below
/*foreach (JProperty jp in jobj.Properties())
* {
* //string tmpPropOut = @"";
* string tmpPropName = jp.Name;
* string tmpPropType = jp.Value.Type.ToString();
* //string tmpPropVal = @"";
* object tmpPropVal = null;
* Type tmpTypeType = Type.GetType(tmpPropType.ToLower());
*
* if (tmpPropType == "Object" || tmpPropType == "Array")
* {
* tmpPropVal = new object();
* tmpPropVal = jp.Value;
*
* if (jp.Value != null)
* {
* string propPrefix = tmpPropName;
* JObject jo = (JObject)tmpPropVal;
* foreach (JProperty jpp in jo.Properties())
* {
*
* }
* }
* }
* else if (tmpPropType == "Null")
* {
* tmpPropVal = @"none";
* }
* else
* {
* tmpPropVal = jp.Value.ToString();
* }
*
* //tmpPropVal = jp.Value;
*
* JsonField jf = new JsonField(tmpFieldCounter, tmpPropName, tmpPropType, null);
* switch (tmpPropType)
* {
* case "String":
* //jf.Value = (string)tmpPropVal;
* jf.Value = (string)((JValue)tmpPropVal.ToString());
* break;
* case "Null":
* //jf.Value = (string)((JValue)tmpPropVal.ToString());
* jf.Value = "none";
* break;
* case "Object":
* jf.Value = new object();
* jf.Value = (object)tmpPropVal;
* break;
* case "Int":
* jf.Value = new int();
* jf.Value = (int)tmpPropVal;
* break;
* default:
* jf.Value = new object();
* jf.Value = (object)tmpPropVal;
* break;
* }
* tmpFields.Add(tmpFieldCounter, jf);
*
* tmpFieldOrder.Add(tmpFieldCounter, jp.Name);
* tmpFieldCounter++;
* }*/
// ========== Property Enumeration/Processing, rev 2: Current 8-31-20
foreach (JProperty jp in jobj.Properties())
{
JsonField jf = new JsonField(tmpFieldCounter, jp.Name, jp.Value.Type.ToString(), null);
switch (jp.Value.Type.ToString())
{
case "String":
jf.Value = (string)((JValue)jp.Value.ToString());
break;
case "Null":
jf.Value = "null";
break;
case "Object":
jf.Value = new object();
jf.Value = (object)jp.Value;
break;
case "Int":
jf.Value = new int();
jf.Value = (int)jp.Value;
break;
default:
jf.Value = new object();
jf.Value = (object)jp.Value;
break;
}
tmpFields.Add(tmpFieldCounter, jf);
tmpFieldOrder.Add(tmpFieldCounter, jp.Name);
tmpFieldCounter++;
}
je.ImportFields(tmpFields);
sw.Stop();
Util.DateLog(String.Format(@"Conversion time: {0}", sw.Elapsed));
return(je);
}