private List <KeyValuePair <string, List <string> > > findDuc(string key) //if (!cat.Contains(t.Key)) is false
{
List <KeyValuePair <string, List <string> > > result = new List <KeyValuePair <string, List <string> > >();
List <string> tbs = new List <string>();
foreach (KeyValuePair <string, string> x in rela)
{
if (ParseJSONObjects.repSpac2Und((x.Value as string)) == ParseJSONObjects.repSpac2Und(key))
{
tbs.Add(ParseJSONObjects.repSpac2Und(x.Key));
}
}
foreach (string x in tbs)
{
//Assume x is Table_#
foreach (KeyValuePair <string, List <string> > y in cates)
{
if (ParseJSONObjects.repSpac2Und(x) == ParseJSONObjects.repSpac2Und(y.Key))
{
result.Add(y);
}
}
}
return(result);
}
private void sortMeow()
{
List <string> rm = new List <string>();
foreach (KeyValuePair <List <string>, List <string> > x in allAttr)
{
foreach (string y in (x.Key as List <string>))
{
foreach (string z in (x.Value as List <string>))
{
if (!oldMeow.ContainsKey(ParseJSONObjects.repSpac2Und(y)))
{
oldMeow.Add(ParseJSONObjects.repSpac2Und(y), new List <string>());
newMeow.Add(ParseJSONObjects.repSpac2Und(y), new List <string>());
}
//You can uncomment, but I don't understand why the Cats.sql file is even more complete. Big Question Mark...
//if ((oldMeow[ParseJSONObjects.repSpac2Und(y)] as List<string>).Contains(ParseJSONObjects.repSpac2Und(z)))
//{
// if (!rm.Contains(ParseJSONObjects.repSpac2Und(z)))
// {
// rm.Add(ParseJSONObjects.repSpac2Und(z));
// }
//}
}
}
foreach (string k in rm)
{
(x.Value as List <string>).Remove(ParseJSONObjects.repSpac2Und((k)));
}
Console.WriteLine("orph atters " + (x.Value as List <string>).Count);
if ((x.Value as List <string>).Count > 0)
{
foreach (string y in (x.Key as List <string>))
{
foreach (string z in (x.Value as List <string>))
{
if (!(oldMeow[ParseJSONObjects.repSpac2Und(y)] as List <string>).Contains(ParseJSONObjects.repSpac2Und(z)))
{
(oldMeow[ParseJSONObjects.repSpac2Und(y)] as List <string>).Add(ParseJSONObjects.repSpac2Und(z));
if (!newMeow.ContainsKey(ParseJSONObjects.repSpac2Und(y)))
{
newMeow.Add(ParseJSONObjects.repSpac2Und(y), new List <string>());
}
(newMeow[ParseJSONObjects.repSpac2Und(y)] as List <string>).Add(ParseJSONObjects.repSpac2Und(z));
}
}
}
}
}
MeowMeow(oldMeow);
}
public void parseJSONFile(string jsonInput, string sqlOutput)
{
int counter;
string line;
System.IO.StreamReader jsonfile;
System.IO.StreamWriter outputfile;
try
{
ParseJSONObjects json2db = new ParseJSONObjects();
Console.WriteLine("\nCreating: " + sqlOutput);
Console.Write("Progress:");
// Read the json data jsonfile.
jsonfile = new System.IO.StreamReader(jsonInput);
// Create the output file
outputfile = new System.IO.StreamWriter(sqlOutput);
counter = 0;
while ((line = jsonfile.ReadLine()) != null)
{
JsonObject my_jsonStr = (JsonObject)JsonObject.Parse(line);
string type = my_jsonStr["type"].ToString();
switch (type)
{
case "\"review\"":
outputfile.WriteLine(json2db.ProcessReviews(my_jsonStr));
break;
case "\"user\"":
outputfile.WriteLine(json2db.ProcessUsers(my_jsonStr));
break;
case "\"business\"":
outputfile.WriteLine(json2db.ProcessBusiness(my_jsonStr));
break;
default: Console.WriteLine("Unknown type : " + type);
break;
}
if ((counter % 5000) == 0)
{
Console.Write("■");
}
counter++;
}
jsonfile.Close();
outputfile.Close();
}
catch (Exception e)
{
Console.Write("Exception:");
Console.WriteLine(e.Message);
}
// Suspend the screen.
Console.WriteLine("\n" + sqlOutput + ": created. \n\n Press a key to continue.");
Console.ReadLine();
}
public void parseJSONFile(string jsonInput, string sqlOutput)
{
int counter;
string line;
System.IO.StreamReader jsonfile;
System.IO.StreamWriter outputfile;
try
{
ParseJSONObjects json2db = new ParseJSONObjects();
Console.WriteLine("\nCreating: " + sqlOutput );
Console.Write("Progress:");
// Read the json data jsonfile.
jsonfile = new System.IO.StreamReader(jsonInput);
// Create the output file
outputfile = new System.IO.StreamWriter(sqlOutput);
counter = 0;
while ((line = jsonfile.ReadLine()) != null)
{
JsonObject my_jsonStr = (JsonObject)JsonObject.Parse(line);
string type = my_jsonStr["type"].ToString();
switch (type)
{
case "\"review\"":
outputfile.WriteLine(json2db.ProcessReviews(my_jsonStr));
break;
case "\"user\"":
outputfile.WriteLine(json2db.ProcessUsers(my_jsonStr));
break;
case "\"business\"":
outputfile.WriteLine(json2db.ProcessBusiness(my_jsonStr));
break;
default: Console.WriteLine("Unknown type : " + type);
break;
}
if ((counter % 5000) == 0)
Console.Write("■");
counter++;
}
jsonfile.Close();
outputfile.Close();
}
catch (Exception e)
{
Console.Write("Exception:");
Console.WriteLine(e.Message);
}
// Suspend the screen.
Console.WriteLine("\n"+sqlOutput+": created. \n\n Press a key to continue.");
Console.ReadLine();
}
private string SuperClean(string inStr)
{
string t = ParseJSONObjects.cleanTextforSQL(inStr);
string s = "";
foreach (char x in t)
{
if (x != '?')
{
s += x;
}
}
return(s);
}
private List <KeyValuePair <string, List <string> > > findsDuc(KeyValuePair <string, Hashtable> keys)
{
List <KeyValuePair <string, List <string> > > t = new List <KeyValuePair <string, List <string> > >();
List <string> tmp = new List <string>();
foreach (string k in keys.Value.Keys)
{
tmp.Add(ParseJSONObjects.repSpac2Und(k));
}
foreach (KeyValuePair <string, List <string> > x in cates)
{
List <string> y = (x.Value as List <string>);
bool ex = true;
foreach (string z in y)
{
if (!tmp.Contains(ParseJSONObjects.repSpac2Und(z)))
{
ex = false;
}
//else
//{
//tmp.Remove(ParseJSONObjects.repSpac2Und(z));
//}
}
if (ex)
{
t.Add(x);
string input = keys.Key;
if (input != null)
{
rela.Add(new KeyValuePair <string, string>(ParseJSONObjects.repSpac2Und(x.Key), ParseJSONObjects.repSpac2Und(input)));
}
}
}
/*
* if (tmp.Count > 0)
* {
* Console.WriteLine("You failed");
* }
*/
return(t);
}
public void Push(List <string> tmp2, Hashtable tmp, string bns)
{
List <string> catt = new List <string>();
List <KeyValuePair <string, Hashtable> > jsnObj = new List <KeyValuePair <string, Hashtable> >();
List <string> recorded = new List <string>();
foreach (string keys in tmp2)
{
string _keysValue = ParseJSONObjects.repSpac2Und(keys.ToString());
if (ParseJSONObjects.Categoriess.Contains(_keysValue))
{
//Console.WriteLine(_keysValue);
if (!catt.Contains(ParseJSONObjects.repSpac2Und(_keysValue)))
{
catt.Add(ParseJSONObjects.repSpac2Und(_keysValue));
}
}
KeyValuePair <string, Hashtable> top = new KeyValuePair <string, Hashtable>(_keysValue, new Hashtable());
ICollection temp = findPre(_keysValue);
if (temp != null)
{
foreach (string k in temp)
{
if (!recorded.Contains(k))
{
recorded.Add(k);
}
if (tmp.ContainsKey(k))
{
if (!(top.Value as Hashtable).ContainsKey(k))
{
(top.Value as Hashtable).Add(k, tmp[k]);
}
else
{
(top.Value as Hashtable)[k] = tmp[k];
}
}
}
}
else
{
//add all values to jsnObj
foreach (string k in tmp.Keys)
{
if (!recorded.Contains(k))
{
recorded.Add(k);
}
(top.Value as Hashtable)[k] = tmp[k];
}
}
jsnObj.Add(top);
}
//For the unused/un-recorded parameters:
List <string> unused = new List <string>();
foreach (string s in tmp.Keys)
{
if (!recorded.Contains(s))
{
unused.Add(s);
}
}
if (unused.Count > 0)
{
//add all unused values to jsnObj
foreach (string k in unused)
{
KeyValuePair <string, Hashtable> tops = new KeyValuePair <string, Hashtable>(k, new Hashtable());
if (!(tops.Value as Hashtable).ContainsKey(ParseJSONObjects.repSpac2Und(k)))
{
(tops.Value as Hashtable).Add(ParseJSONObjects.repSpac2Und(k), tmp[k]);
}
else
{
(tops.Value as Hashtable)[ParseJSONObjects.repSpac2Und(k)] = tmp[k];
}
jsnObj.Add(tops);
}
}
KeyValuePair <string, Hashtable> sth;
stack.Push(new KeyValuePair <string, List <KeyValuePair <string, Hashtable> > >(bns, jsnObj));
if (catt.Count() == 1)
{
sth = new KeyValuePair <string, Hashtable>(ParseJSONObjects.repSpac2Und(catt.First()), tmp);
if (!oldMeow.ContainsKey(ParseJSONObjects.repSpac2Und(catt.First())))
{
oldMeow.Add(ParseJSONObjects.repSpac2Und(catt.First()), new List <string>());
}
foreach (string k in tmp.Keys)
{
if (!(oldMeow[ParseJSONObjects.repSpac2Und(catt.First())] as List <string>).Contains(ParseJSONObjects.repSpac2Und(k)))
{
(oldMeow[ParseJSONObjects.repSpac2Und(catt.First())] as List <string>).Add(ParseJSONObjects.repSpac2Und(k));
}
}
chunk.Push(new KeyValuePair <string, KeyValuePair <string, Hashtable> >(bns, sth));
}
else if (catt.Count() > 1)
{
sth = new KeyValuePair <string, Hashtable>(null, tmp);
List <string> meowAttr = new List <string>();
foreach (string k in tmp.Keys)
{
if (!meowAttr.Contains(ParseJSONObjects.repSpac2Und(k)))
{
meowAttr.Add(ParseJSONObjects.repSpac2Und(k));
}
}
KeyValuePair <List <string>, List <string> > meow = new KeyValuePair <List <string>, List <string> >(catt, meowAttr);
allAttr.Add(meow);
chunk.Push(new KeyValuePair <string, KeyValuePair <string, Hashtable> >(bns, sth));
}
}
public void Pop()
{
foreach (KeyValuePair <string, List <KeyValuePair <string, Hashtable> > > tmpp in stack)
{
List <KeyValuePair <string, Hashtable> > tmp = tmpp.Value as List <KeyValuePair <string, Hashtable> >;
foreach (KeyValuePair <string, Hashtable> t in tmp)
{
List <KeyValuePair <string, List <string> > > temp;
if (!cat.Contains(ParseJSONObjects.repSpac2Und(t.Key)))
{
cat.Add(ParseJSONObjects.repSpac2Und(t.Key));
temp = asemb(t);
}
else
{
temp = findDuc(ParseJSONObjects.repSpac2Und(t.Key));
}
}
}
rela.Clear();
foreach (KeyValuePair <string, KeyValuePair <string, Hashtable> > tmpp in chunk)
{
List <string> tables = new List <string>();
KeyValuePair <string, Hashtable> tmp = tmpp.Value;
//If cat does not contain the category
List <KeyValuePair <string, List <string> > > temp = findsDuc(tmp);
//write mysql query of saved cates and required parameters as recorded, and save those queries to local *.sql file.
foreach (KeyValuePair <string, List <string> > product in temp)
{
if (!tables.Contains(ParseJSONObjects.repSpac2Und(product.Key)))
{
tables.Add(ParseJSONObjects.repSpac2Und(product.Key));
string query = "Insert into " + ParseJSONObjects.repSpac2Und(product.Key.ToString()) + " set business_id='" + ParseJSONObjects.cleanTextforSQL(tmpp.Key) + "',";
int before = query.Length;
foreach (string s in (product.Value as List <string>))
{
query += ParseJSONObjects.repSpac2Und(s) + "='" + ParseJSONObjects.cleanTextforSQL((tmp.Value as Hashtable)[ParseJSONObjects.repSpac2Und(s)].ToString()) + "',";
}
query = query.Substring(0, query.Length - 1) + ";";
using (StreamWriter w = File.AppendText(Parser.dataDir + "/sub/insertIntoCatTables.sql"))
{
w.WriteLine(query);
}
}
//end inner 'foreach'
}
//end outter 'foreach'
}
this.sortMeow();
//end func
}
private List <KeyValuePair <string, List <string> > > asemb(KeyValuePair <string, Hashtable> input) //if (!cat.Contains(t.Key)) is true
{
int i = 0;
Hashtable hash = input.Value as Hashtable;
List <KeyValuePair <string, List <string> > > t = new List <KeyValuePair <string, List <string> > >();
List <string> tmp = new List <string>();
List <KeyValuePair <string, List <string> > > cates_add = new List <KeyValuePair <string, List <string> > >();
List <KeyValuePair <string, List <string> > > cates_remove = new List <KeyValuePair <string, List <string> > >();
foreach (string x in hash.Keys)
{
tmp.Add(ParseJSONObjects.repSpac2Und(x));
}
foreach (KeyValuePair <string, List <string> > x in cates)
{
List <string> y = (x.Value as List <string>);
List <string> anoProduct = new List <string>();
bool ex = true;
foreach (string z in y)
{
if (!tmp.Contains(ParseJSONObjects.repSpac2Und(z)))
{
ex = false;
}
else
{
tmp.Remove(ParseJSONObjects.repSpac2Und(z));
anoProduct.Add(ParseJSONObjects.repSpac2Und(z));
}
}
if (ex)
{
t.Add(x);
rela.Add(new KeyValuePair <string, string>(ParseJSONObjects.repSpac2Und(x.Key), ParseJSONObjects.repSpac2Und(input.Key)));
}
else if (anoProduct.Count > 0 & anoProduct.Count < (x.Value as List <string>).Count)
{
//Breakdown the prodcut into assemb;
cates_remove.Add(x);
KeyValuePair <string, List <string> > new_x = new KeyValuePair <string, List <string> >(ParseJSONObjects.repSpac2Und(x.Key), anoProduct);
cates_add.Add(new_x);
List <string> ano = new List <string>();
foreach (string xs in x.Value)
{
if (!anoProduct.Contains(ParseJSONObjects.repSpac2Und(xs)))
{
ano.Add(ParseJSONObjects.repSpac2Und(xs));
}
}
if (ano.Count > 0)
{
string newName = "n" + ParseJSONObjects.repSpac2Und(x.Key) + j.ToString();
j++;
KeyValuePair <string, List <string> > new_s = new KeyValuePair <string, List <string> >(ParseJSONObjects.repSpac2Und(newName), ano);
cates_add.Add(new_s);
List <KeyValuePair <string, string> > tbs = new List <KeyValuePair <string, string> >();
foreach (KeyValuePair <string, string> xs in rela)
{
if (ParseJSONObjects.repSpac2Und((xs.Key as string)) == ParseJSONObjects.repSpac2Und(x.Key))
{
tbs.Add(new KeyValuePair <string, string>(ParseJSONObjects.repSpac2Und(newName), ParseJSONObjects.repSpac2Und(xs.Value)));
}
}
foreach (KeyValuePair <string, string> xs in tbs)
{
rela.Add(xs);
}
}
}
i++;
}
foreach (KeyValuePair <string, List <string> > sth in cates_remove)
{
cates.Remove(sth);
}
foreach (KeyValuePair <string, List <string> > sth in cates_add)
{
cates.Add(sth);
}
List <string> ts = new List <string>();
foreach (string k in tmp)
{
ts.Add(ParseJSONObjects.repSpac2Und(k));
}
if (ts.Count > 0)
{
KeyValuePair <string, List <string> > tt = new KeyValuePair <string, List <string> >(ParseJSONObjects.repSpac2Und("Table_" + i.ToString()), ts);
cates.Add(tt);
rela.Add(new KeyValuePair <string, string>(ParseJSONObjects.repSpac2Und(tt.Key), ParseJSONObjects.repSpac2Und(input.Key)));
t.Add(tt);
}
return(t);
}
private void MeowMeow(Hashtable tb)
{
Console.WriteLine("rela has : " + rela.Count);
Console.WriteLine("oldMeow has : " + oldMeow.Count);
Console.WriteLine("newMeow has : " + newMeow.Count);
foreach (string m in tb.Keys)
{
List <string> tmp = new List <string>();
foreach (string k in (tb[ParseJSONObjects.repSpac2Und(m)] as List <string>))
{
tmp.Add(ParseJSONObjects.repSpac2Und(k));
}
foreach (KeyValuePair <string, List <string> > x in cates)
{
//if (x.Key == m)
//{
List <string> y = (x.Value as List <string>);
bool ex = true;
foreach (string z in y)
{
if (!tmp.Contains(ParseJSONObjects.repSpac2Und(z)))
{
ex = false;
break;
}
/*
* else
* {
* tmp.Remove(ParseJSONObjects.repSpac2Und(z));
* }
*/
}
if (ex)
{
Console.WriteLine("Your ex-!!");
rela.Add(new KeyValuePair <string, string>(ParseJSONObjects.repSpac2Und(x.Key), ParseJSONObjects.repSpac2Und(m)));
}
//}
}
if (tmp.Count > 0)
{
Console.WriteLine("You failed");
}
}
Console.WriteLine("First blood!!");
foreach (KeyValuePair <string, string> r in rela)
{
string query = "Insert into Cats set ";
query += "Parent='" + r.Key + "', ";
query += "Child='" + r.Value + "'; ";
using (StreamWriter w = File.AppendText(Parser.dataDir + "/sub/Cats.sql"))
{
w.WriteLine(query);
}
}
}
