/// <summary>
/// Initializes the journal dictionary.
/// </summary>
private void InitializeJournalDictionary()
{
JSONNode json = JSON_IO.ReadJSON("journalItems");
journalTable = new Hashtable();
for (int i = 0; i < json.Count; i++) //Iteract through phases in the .json
{
ExperimentsState.Stage[] setOfStages = new ExperimentsState.Stage[json[i].Count];
for (int j = 0; j < json[i].Count; j++) //Iteract through the stages in the phase i
{
setOfStages [j] = new ExperimentsState.Stage();
setOfStages [j].name = json [i][j] ["name"];
setOfStages [j].steps = new ExperimentsState.JournalStep[json [i][j]["steps"].Count];
for (int k = 0; k < json [i][j] ["steps"].Count; k++) //Iteract through the steps of stage j
{
setOfStages [j].steps [k] = new ExperimentsState.JournalStep();
setOfStages [j].steps [k].text = json [i][j] ["steps"][k]["text"].Value;
if (json [i] [j] ["steps"] [k] ["isDone"].AsBool != null)
{
setOfStages [j].steps [k].isDone = json [i] [j] ["steps"] [k] ["isDone"].AsBool;
}
}
}
journalTable [i] = setOfStages;
}
}
//
private static void PopulateAllProfessionNearestNeighbors(int n)
{
MongoClient dbClient = new MongoClient(MongoStrings.CONNECTION);
IMongoDatabase database = dbClient.GetDatabase(MongoStrings.GRAPH_DB);
var edge_collection = database.GetCollection <BsonDocument>("edges_professions");
var destination_collection = database.GetCollection <BsonDocument>("nearest_neighbors_profession");
var ForkReport = new ONETReport();
ForkReport.MasterOccupationList = JSON_IO.Import_OccupationList(Helper.Publics.FILENAMES.OCCUPATIONS + ".txt");
Console.WriteLine("Iterating Through Occupations...");
foreach (Occupation o in ForkReport.MasterOccupationList)
{
Console.WriteLine(" ..." + o.Name);
var occupationFilterA = Builders <BsonDocument> .Filter.Eq("OccupationAName", o.Name);
var targetAdjacenciesA = edge_collection.Find(occupationFilterA).ToList();
var occupationFilterB = Builders <BsonDocument> .Filter.Eq("OccupationBName", o.Name);
var targetAdjacenciesB = edge_collection.Find(occupationFilterB).ToList();
var MasterAdjacencyList = new List <SimpleOccupationEdge>();
//retrieve adjacencies from database
foreach (var a in targetAdjacenciesA)
{
MasterAdjacencyList.Add(BsonSerializer.Deserialize <SimpleOccupationEdge>(a));
}
foreach (var b in targetAdjacenciesB)
{
MasterAdjacencyList.Add(BsonSerializer.Deserialize <SimpleOccupationEdge>(b));
}
ProfessionNearestNeighbors newNeighborList = new ProfessionNearestNeighbors(o.Name);
//sort adjacencies
foreach (AttributeType foo in Enum.GetValues(typeof(AttributeType)))
{
if (foo == AttributeType.Word2VecWIKI || foo == AttributeType.Skill || foo == AttributeType.Net || foo == AttributeType.Knowledge || foo == AttributeType.Ability)
{
List <SimpleOccupationEdge> SortedAdjacencyList = MasterAdjacencyList.OrderBy(oy => oy.getDistance(foo)).ToList();
var newList = SortedAdjacencyList.Take(n).Select(c => c.getOtherName(o.Name)).ToList();
newNeighborList.NearestNeighbors.Add(new Tuple <AttributeType, List <string> >(foo, newList));
}
}
destination_collection.InsertOne(newNeighborList.ToBsonDocument());
}
}
//ETL the ONET data from origial ONET dowloadable db format to MongoDB
private static void ONET_ETL_NoProfession()
{
var ForkReport = new ONETReport();
ForkReport.MasterOccupationList = JSON_IO.Import_OccupationList(Helper.Publics.FILENAMES.OCCUPATIONS + ".txt");
ForkReport.MasterSkillList = JSON_IO.Import_AttributeList(Helper.Publics.FILENAMES.SKILLS + ".txt");
ForkReport.MasterAbilityList = JSON_IO.Import_AttributeList(Helper.Publics.FILENAMES.ABILITIES + ".txt");
ForkReport.MasterKnowledgeList = JSON_IO.Import_AttributeList(Helper.Publics.FILENAMES.KNOWLEDGE + ".txt");
MongoClient dbClient = new MongoClient(MongoStrings.CONNECTION);
IMongoDatabase database = dbClient.GetDatabase(MongoStrings.GRAPH_DB);
var skill_collection = database.GetCollection <BsonDocument>("node_skill");
var ability_collection = database.GetCollection <BsonDocument>("node_ability");
// var profession_collection = database.GetCollection<BsonDocument>("node_profession");
var knowledge_collection = database.GetCollection <BsonDocument>("node_knowledge");
//add those top 5 occupations
//save to DB
var EmpInfoArray = new List <BsonDocument>();
foreach (Attribute j in ForkReport.MasterSkillList)
{
EmpInfoArray.Add(j.ToBsonDocument());
}
skill_collection.InsertMany(EmpInfoArray);
EmpInfoArray = new List <BsonDocument>();
foreach (Attribute j in ForkReport.MasterKnowledgeList)
{
EmpInfoArray.Add(j.ToBsonDocument());
}
knowledge_collection.InsertMany(EmpInfoArray);
EmpInfoArray = new List <BsonDocument>();
foreach (Attribute j in ForkReport.MasterAbilityList)
{
EmpInfoArray.Add(j.ToBsonDocument());
}
ability_collection.InsertMany(EmpInfoArray);
//EmpInfoArray = new List<BsonDocument>();
/*foreach (Occupation j in ForkReport.MasterOccupationList)
* {
* EmpInfoArray.Add(j.ToBsonDocument());
* }
* profession_collection.InsertMany(EmpInfoArray);
*/
}
public Form1()
{
InitializeComponent();
JSONFactory.JSON_IO JIO = new JSON_IO(Utils.Utils.this_Path() + "\\TestTubeObjects.txt");
string jsondata=JIO.JSON_Read();
//JSONFactory.JSONObjects settings;
Exception ee;
if (!JIO.JSON_Deserialize<JSONFactory.JSONObjects>(jsondata, out settings, out ee))
this.labelStatus.Text = ee.Message;
else
Init();
}
/// <summary>
/// Loads the stages.
/// </summary>
/// <returns>Array of stages loaded.</returns>
public static Stage[] LoadStages () {
JSONNode json = JSON_IO.ReadJSON ("journalItems");
Stage[] stages = new Stage[json.Count];
for (int i = 0; i < json.Count; i++) {
stages [i] = new Stage ();
stages [i].name = json [i] ["name"];
stages [i].steps = new JournalStep[json [i] ["steps"].Count];
for (int j = 0; j < json [i] ["steps"].Count; j++) {
stages [i].steps [j] = new JournalStep ();
stages [i].steps [j].text = json [i] ["steps"][j]["text"].Value;
stages [i].steps [j].isDone = json [i] ["steps"][j]["isDone"].AsBool;
}
}
return stages;
}
private bool FindTest()
{
bool rc = false;
JSONIO.JSON_IO jio = new JSON_IO();
Exception e;
if (jio.JSON_Deserialize<JSONIO.SUITE>(this.JSON, out mTestCase, out e))
{
updateStatus(StatusType.FOUND, mTestCase.total_count.ToString());
}
else
{
this.LastError = e;
if(this.LastError!=null)
updateStatus(StatusType.ERROR, e.Message);
else
updateStatus(StatusType.FAILED, "Not found");
}
return rc;
}
//ETL the profession info, add top adjacencies to profession node (this part is bad and needs to go)
private static void ONET_ETL_Profession()
{
var ForkReport = new ONETReport();
ForkReport.MasterOccupationList = JSON_IO.Import_OccupationList(Helper.Publics.FILENAMES.OCCUPATIONS + ".txt");
ForkReport.MasterSkillList = JSON_IO.Import_AttributeList(Helper.Publics.FILENAMES.SKILLS + ".txt");
ForkReport.MasterAbilityList = JSON_IO.Import_AttributeList(Helper.Publics.FILENAMES.ABILITIES + ".txt");
ForkReport.MasterKnowledgeList = JSON_IO.Import_AttributeList(Helper.Publics.FILENAMES.KNOWLEDGE + ".txt");
MongoClient dbClient = new MongoClient(MongoStrings.CONNECTION);
IMongoDatabase database = dbClient.GetDatabase(MongoStrings.GRAPH_DB);
var profession_collection = database.GetCollection <BsonDocument>("node_profession");
//add those top 5 occupations
var collection = database.GetCollection <BsonDocument>("edges_professions");
foreach (Occupation o in ForkReport.MasterOccupationList)
{
// var occupationFilter = Builders<BsonDocument>.Filter.ElemMatch<BsonValue>(
// "OccupationAName",
//
// );
var occupationFilterA = Builders <BsonDocument> .Filter.Eq("OccupationAName", o.Name);
var targetAdjacenciesA = collection.Find(occupationFilterA).ToList();
var occupationFilterB = Builders <BsonDocument> .Filter.Eq("OccupationBName", o.Name);
var targetAdjacenciesB = collection.Find(occupationFilterB).ToList();
var MasterAdjacencyList = new List <SimpleOccupationEdge>();
//retrieve adjacencies from database
foreach (var a in targetAdjacenciesA)
{
MasterAdjacencyList.Add(BsonSerializer.Deserialize <SimpleOccupationEdge>(a));
}
foreach (var b in targetAdjacenciesB)
{
MasterAdjacencyList.Add(BsonSerializer.Deserialize <SimpleOccupationEdge>(b));
}
//sort adjacencies
List <SimpleOccupationEdge> SortedAdjacencyList = MasterAdjacencyList.OrderBy(oy => oy.getDistance()).ToList();
//save top adjacencies
//REMOVING - NOT HOW WE ARE MANAGING THIS INFO
//o.TenMostAdjacent = SortedAdjacencyList.Take(10).Select(c => c.getOtherName(o.Name)).ToList();
}
//save to DB
var EmpInfoArray = new List <BsonDocument>();
foreach (Occupation j in ForkReport.MasterOccupationList)
{
EmpInfoArray.Add(j.ToBsonDocument());
}
profession_collection.InsertMany(EmpInfoArray);
}