private void PrintModificationMenu(Data data, List <Dictionary <String, Object> > meta) { Console.WriteLine("Select item by is position :"); int i = 0; List <String> keys = new List <string>(); foreach (var item in meta) { keys.Add(i.ToString()); Console.WriteLine(i); i++; } Int32 selectedItem; Int32.TryParse(Console.ReadLine(), out selectedItem); String choosed = ConsoleManager.Instance.LoopSinceChoseOrExit(keys, "exit"); Object dicoItem = JsonDataExtractor.Instance("json1").GetSubListObjectById(meta, choosed); if (dicoItem is Dictionary <String, Object> ) { PrintModificationMenu(data, dicoItem as Dictionary <String, Object>); } else { PrintModificationMenu(data, dicoItem as List <Dictionary <String, Object> >); } }
public void PropertiesBuiltInExtractsWholeDocument() { var json = JsonConvert.SerializeObject(new { Person = new { Name = "Nick" } }); var extractor = new JsonDataExtractor("@Properties"); var value = extractor.ExtractData(new StringReader(json)); var rt = JsonConvert.SerializeObject(value); Assert.Equal(json, rt); }
public void ExtractsNestedJsonValues() { var json = JsonConvert.SerializeObject(new { Person = new { Name = "Nick" } }); var extractor = new JsonDataExtractor("Person.Name"); var value = extractor.ExtractData(new StringReader(json)); var jv = Assert.IsType <JValue>(value); Assert.Equal("Nick", jv.Value); }
private void PrintModification(Data data) { Object meta = JsonDataExtractor.Instance("json1").Init(data.JsonData).BaseJson; ConsoleManager.Instance.LoopSinceExit("exit", () => { PrintModificationMenu(data, meta as Dictionary <String, Object>); }); }
private Dictionary <String, Object> FieldRemover(Dictionary <String, Object> meta) { Console.WriteLine("Select key to remove"); foreach (var item in meta) { Console.WriteLine(item.Key); } String keyToRemove = ConsoleManager.Instance.LoopSinceChoseOrExit(meta.Keys, "exit"); return(JsonDataExtractor.Instance("json1").FieldRemover(keyToRemove)); }
public static DataSet ToParquetDataSet(this JObject jObject, PSchema schema) { if (schema == null) { throw new ArgumentNullException(nameof(schema)); } //convert data var dataExtractor = new JsonDataExtractor(schema); var ds = new DataSet(schema); dataExtractor.AddRow(ds, jObject); return(ds); }
public static DataSet ToParquetDataSet(this JObject jObject) { //extract schema var schemaExtractor = new JsonSchemaExtractor(); schemaExtractor.Analyze(jObject); PSchema schema = schemaExtractor.GetSchema(); //convert data var dataExtractor = new JsonDataExtractor(schema); var ds = new DataSet(schema); dataExtractor.AddRow(ds, jObject); return(ds); }
public void Read_multiple_with_missing_data() { var schema = new Schema( new DataField <int?>("id"), new DataField <string>("country"), new StructField("population", new DataField <int?>("year"), new DataField <int?>("amount"), new DataField <int?>("diff")), new DataField <string>("comment")); var extractor = new JsonDataExtractor(schema); JObject doc1 = JObject.Parse(ReadJson("infer00.json")); JObject doc2 = JObject.Parse(ReadJson("infer01.json")); var ds = new DataSet(schema); extractor.AddRow(ds, doc1); extractor.AddRow(ds, doc2); Assert.Equal(2, ds.RowCount); Assert.Equal("{123;UK;{2016;111;<null>};<null>}", ds[0].ToString()); Assert.Equal("{123;UK;{2017;222;111};no comments}", ds[1].ToString()); }
private Dictionary <String, Object> FieldAdder(Dictionary <String, Object> meta) { ConsoleManager.Instance.LoopSinceExit("Continue ? Y/N", "N", () => { Console.WriteLine("Existing keys :"); foreach (var item in meta) { Console.Write(item.Key + ", "); } Console.WriteLine(); Console.WriteLine("New key to create :"); String key = ConsoleManager.Instance.LinePromptSaver(); Console.WriteLine("New value for key " + key + " :"); String value = ConsoleManager.Instance.LinePromptSaver(); meta = JsonDataExtractor.Instance("json1").FieldAdder(key, value); }); return(meta); }
//[Fact] public void TempTest() { var dir = new DirectoryInfo(@"C:\Users\ivang\Downloads\Fullfeed-20170330004044"); FileInfo[] files = dir.GetFiles(); JObject[] jos = files .Select(fi => JObject.Parse(System.IO.File.ReadAllText(fi.FullName))) .Take(1000) .ToArray(); var inferrer = new JsonSchemaInferring(); Schema schema = inferrer.InferSchema(jos); var extractor = new JsonDataExtractor(schema); var ds = new DataSet(schema); for (int i = 0; i < jos.Length; i++) { extractor.AddRow(ds, jos[i]); } ParquetWriter.WriteFile(ds, "c:\\tmp\\com.parquet"); }
private async void PrintModificationMenu(Data data, Dictionary <String, Object> meta) { Console.WriteLine("Object structure :"); foreach (var item in meta as Dictionary <String, Object> ) { Console.WriteLine(item.Key + " : " + item.Value); } Console.WriteLine(); Console.WriteLine("1 : remove field"); Console.WriteLine("2 : add field"); Console.WriteLine("3 : update field"); Console.WriteLine("4 : remove current object"); if (JsonDataExtractor.Instance("json1").HaveSubObject(meta)) { Console.WriteLine("5 : navigate to object"); } else { Console.WriteLine("9 : validate"); } Console.WriteLine("exit to quit"); Int32 selectedItem; Int32.TryParse(Console.ReadLine(), out selectedItem); switch (selectedItem) { case 1: meta = FieldRemover(meta); String metaJson = JsonDataExtractor.Instance("json1").SendToJson(meta); break; case 2: meta = FieldAdder(meta); break; case 3: break; case 5: List <String> keys = JsonDataExtractor.Instance("json1").GetSubObjectsKeys(meta); Console.WriteLine("Avaibles keys :"); foreach (var item in keys) { Console.WriteLine("- " + item); } String choosed = ConsoleManager.Instance.LoopSinceChoseOrExit(keys, "exit"); Object dicoItem = JsonDataExtractor.Instance("json1").GetSubObjectByKey(meta, choosed); if (dicoItem is Dictionary <String, Object> ) { PrintModificationMenu(data, dicoItem as Dictionary <String, Object>); } else { PrintModificationMenu(data, dicoItem as List <Dictionary <String, Object> >); } break; case 9: String result = JsonDataExtractor.Instance("json1").SendBaseJsonToString(); if (result != null) { data.JsonData = result; await dataManager.Update(data); } else { Console.WriteLine("Json is corrupted!"); } break; default: break; } }