private void PrintModificationMenu(Data data, List <Dictionary <String, Object> > meta)
{
Console.WriteLine("Select item by is position :");
int i = 0;
List <String> keys = new List <string>();
foreach (var item in meta)
{
keys.Add(i.ToString());
Console.WriteLine(i);
i++;
}
Int32 selectedItem;
Int32.TryParse(Console.ReadLine(), out selectedItem);
String choosed = ConsoleManager.Instance.LoopSinceChoseOrExit(keys, "exit");
Object dicoItem = JsonDataExtractor.Instance("json1").GetSubListObjectById(meta, choosed);
if (dicoItem is Dictionary <String, Object> )
{
PrintModificationMenu(data, dicoItem as Dictionary <String, Object>);
}
else
{
PrintModificationMenu(data, dicoItem as List <Dictionary <String, Object> >);
}
}
public void PropertiesBuiltInExtractsWholeDocument()
{
var json = JsonConvert.SerializeObject(new { Person = new { Name = "Nick" } });
var extractor = new JsonDataExtractor("@Properties");
var value = extractor.ExtractData(new StringReader(json));
var rt = JsonConvert.SerializeObject(value);
Assert.Equal(json, rt);
}
public void ExtractsNestedJsonValues()
{
var json = JsonConvert.SerializeObject(new { Person = new { Name = "Nick" } });
var extractor = new JsonDataExtractor("Person.Name");
var value = extractor.ExtractData(new StringReader(json));
var jv = Assert.IsType <JValue>(value);
Assert.Equal("Nick", jv.Value);
}
private void PrintModification(Data data)
{
Object meta = JsonDataExtractor.Instance("json1").Init(data.JsonData).BaseJson;
ConsoleManager.Instance.LoopSinceExit("exit", () =>
{
PrintModificationMenu(data, meta as Dictionary <String, Object>);
});
}
private Dictionary <String, Object> FieldRemover(Dictionary <String, Object> meta)
{
Console.WriteLine("Select key to remove");
foreach (var item in meta)
{
Console.WriteLine(item.Key);
}
String keyToRemove = ConsoleManager.Instance.LoopSinceChoseOrExit(meta.Keys, "exit");
return(JsonDataExtractor.Instance("json1").FieldRemover(keyToRemove));
}
public static DataSet ToParquetDataSet(this JObject jObject, PSchema schema)
{
if (schema == null)
{
throw new ArgumentNullException(nameof(schema));
}
//convert data
var dataExtractor = new JsonDataExtractor(schema);
var ds = new DataSet(schema);
dataExtractor.AddRow(ds, jObject);
return(ds);
}
public static DataSet ToParquetDataSet(this JObject jObject)
{
//extract schema
var schemaExtractor = new JsonSchemaExtractor();
schemaExtractor.Analyze(jObject);
PSchema schema = schemaExtractor.GetSchema();
//convert data
var dataExtractor = new JsonDataExtractor(schema);
var ds = new DataSet(schema);
dataExtractor.AddRow(ds, jObject);
return(ds);
}
public void Read_multiple_with_missing_data()
{
var schema = new Schema(
new DataField <int?>("id"),
new DataField <string>("country"),
new StructField("population",
new DataField <int?>("year"),
new DataField <int?>("amount"),
new DataField <int?>("diff")),
new DataField <string>("comment"));
var extractor = new JsonDataExtractor(schema);
JObject doc1 = JObject.Parse(ReadJson("infer00.json"));
JObject doc2 = JObject.Parse(ReadJson("infer01.json"));
var ds = new DataSet(schema);
extractor.AddRow(ds, doc1);
extractor.AddRow(ds, doc2);
Assert.Equal(2, ds.RowCount);
Assert.Equal("{123;UK;{2016;111;<null>};<null>}", ds[0].ToString());
Assert.Equal("{123;UK;{2017;222;111};no comments}", ds[1].ToString());
}
private Dictionary <String, Object> FieldAdder(Dictionary <String, Object> meta)
{
ConsoleManager.Instance.LoopSinceExit("Continue ? Y/N", "N",
() =>
{
Console.WriteLine("Existing keys :");
foreach (var item in meta)
{
Console.Write(item.Key + ", ");
}
Console.WriteLine();
Console.WriteLine("New key to create :");
String key = ConsoleManager.Instance.LinePromptSaver();
Console.WriteLine("New value for key " + key + " :");
String value = ConsoleManager.Instance.LinePromptSaver();
meta = JsonDataExtractor.Instance("json1").FieldAdder(key, value);
});
return(meta);
}
//[Fact]
public void TempTest()
{
var dir = new DirectoryInfo(@"C:\Users\ivang\Downloads\Fullfeed-20170330004044");
FileInfo[] files = dir.GetFiles();
JObject[] jos = files
.Select(fi => JObject.Parse(System.IO.File.ReadAllText(fi.FullName)))
.Take(1000)
.ToArray();
var inferrer = new JsonSchemaInferring();
Schema schema = inferrer.InferSchema(jos);
var extractor = new JsonDataExtractor(schema);
var ds = new DataSet(schema);
for (int i = 0; i < jos.Length; i++)
{
extractor.AddRow(ds, jos[i]);
}
ParquetWriter.WriteFile(ds, "c:\\tmp\\com.parquet");
}
private async void PrintModificationMenu(Data data, Dictionary <String, Object> meta)
{
Console.WriteLine("Object structure :");
foreach (var item in meta as Dictionary <String, Object> )
{
Console.WriteLine(item.Key + " : " + item.Value);
}
Console.WriteLine();
Console.WriteLine("1 : remove field");
Console.WriteLine("2 : add field");
Console.WriteLine("3 : update field");
Console.WriteLine("4 : remove current object");
if (JsonDataExtractor.Instance("json1").HaveSubObject(meta))
{
Console.WriteLine("5 : navigate to object");
}
else
{
Console.WriteLine("9 : validate");
}
Console.WriteLine("exit to quit");
Int32 selectedItem;
Int32.TryParse(Console.ReadLine(), out selectedItem);
switch (selectedItem)
{
case 1:
meta = FieldRemover(meta);
String metaJson = JsonDataExtractor.Instance("json1").SendToJson(meta);
break;
case 2:
meta = FieldAdder(meta);
break;
case 3:
break;
case 5:
List <String> keys = JsonDataExtractor.Instance("json1").GetSubObjectsKeys(meta);
Console.WriteLine("Avaibles keys :");
foreach (var item in keys)
{
Console.WriteLine("- " + item);
}
String choosed = ConsoleManager.Instance.LoopSinceChoseOrExit(keys, "exit");
Object dicoItem = JsonDataExtractor.Instance("json1").GetSubObjectByKey(meta, choosed);
if (dicoItem is Dictionary <String, Object> )
{
PrintModificationMenu(data, dicoItem as Dictionary <String, Object>);
}
else
{
PrintModificationMenu(data, dicoItem as List <Dictionary <String, Object> >);
}
break;
case 9:
String result = JsonDataExtractor.Instance("json1").SendBaseJsonToString();
if (result != null)
{
data.JsonData = result;
await dataManager.Update(data);
}
else
{
Console.WriteLine("Json is corrupted!");
}
break;
default:
break;
}
}
