private InputEntity GetFramework(object graph)
{
//string ctid = "";
InputEntity entity = new InputEntity();
Newtonsoft.Json.Linq.JArray jarray = (Newtonsoft.Json.Linq.JArray)graph;
foreach (var token in jarray)
{
if (token.GetType() == typeof(Newtonsoft.Json.Linq.JObject))
{
if (token.ToString().IndexOf("ceasn:CompetencyFramework") > -1)
{
entity = ((Newtonsoft.Json.Linq.JObject)token).ToObject <InputEntity>();
//InputEntity cf = ( InputEntity ) JsonConvert.DeserializeObject( token.ToString() );
return(entity);
}
else if (token.ToString().IndexOf("ceasn:Competency") > -1)
{
//ignore
//var c1 = token.ToString().Replace( "exactMatch", "exactAlignment" );
//var c2 = ( ( Newtonsoft.Json.Linq.JObject ) c1 ).ToObject<RA.Models.Json.InputCompetency>();
}
//var itemProperties = token.Children<JProperty>();
////you could do a foreach or a linq here depending on what you need to do exactly with the value
//var myElement = itemProperties.FirstOrDefault( x => x.Name == "url" );
//var myElementValue = myElement.Value; ////This is a JValue type
}
else
{
//error
}
}
//no ctid found, so????
return(entity);
}
public string Import(string startingDate, string endingDate, int maxRecords, string community, bool downloadOnly = false)
{
bool importingThisType = UtilityManager.GetAppKeyValue("importing_competency_frameworks", true);
if (!importingThisType)
{
LoggingHelper.DoTrace(1, "=== ***************** Skipping import of Competency Frameworks ***************** ");
return("Skipped import of competency_frameworks");
}
LoggingHelper.DoTrace(1, string.Format("=== ***************** {0} ***************** ", thisClassName));
JsonEntity input = new JsonEntity();
ReadEnvelope envelope = new ReadEnvelope();
List <ReadEnvelope> list = new List <ReadEnvelope>();
EntityServices mgr = new EntityServices();
string type = "competency_framework";
int pageNbr = 1;
int pageSize = 75;
string importError = "";
string importResults = "";
string importNote = "";
ThisEntity output = new ThisEntity();
List <string> messages = new List <string>();
//the nbr of records needs to be monitored, to determine the optimum
//NOTE: consider the IOER approach that all candidate records are first downloaded, and then a separate process does the import
int cntr = 0;
int pTotalRows = 0;
int exceptionCtr = 0;
string statusMessage = "";
bool isComplete = false;
bool importSuccessfull = true;
int newImportId = 0;
SaveStatus status = new SaveStatus();
//will need to handle multiple calls - watch for time outs
while (pageNbr > 0 && !isComplete)
{
list = RegistryServices.Search(type, startingDate, endingDate, pageNbr, pageSize, ref pTotalRows, ref statusMessage, community);
//list = GetLatest( type, startingDate, endingDate, pageNbr, pageSize, ref pTotalRows, ref statusMessage, community );
if (list == null || list.Count == 0)
{
isComplete = true;
if (pageNbr == 1)
{
importNote = "Competency Frameworks: No records where found for date range ";
//Console.WriteLine( thisClassName + importNote );
LoggingHelper.DoTrace(4, thisClassName + importNote);
}
break;
}
foreach (ReadEnvelope item in list)
{
cntr++;
string payload = item.DecodedResource.ToString();
LoggingHelper.DoTrace(2, string.Format("{0}. EnvelopeIdentifier: {1} ", cntr, item.EnvelopeIdentifier));
status = new SaveStatus();
status.DoingDownloadOnly = downloadOnly;
importError = "";
importSuccessfull = false;
try
{
//Console.WriteLine( string.Format( "{0}. Competency Frameswork Envelope Identifier {1} ", cntr, item.EnvelopeIdentifier ) );
importSuccessfull = entityImportMgr.ProcessEnvelope(mgr, item, status);
}
catch (Exception ex)
{
if (ex.Message.IndexOf("Path '@context', line 1") > 0)
{
importError = "The referenced registry document is using an old schema. Please republish it with the latest schema!";
status.AddError(importError);
}
else
{
LoggingHelper.LogError(ex, string.Format("Exception encountered in envelopeId: {0}", item.EnvelopeIdentifier), false, "CredentialFinder Import exception");
status.AddError(ex.Message);
importError = ex.Message;
LoggingHelper.DoTrace(1, " show exception to use getting related envelopeId " + ex.Message);
}
//make continue on exceptions an option
exceptionCtr++;
if (maxExceptions > 0 && exceptionCtr > maxExceptions)
{
//arbitrarily stop if large number of exceptions
importNote = string.Format(thisClassName + " - Many exceptions ({0}) were encountered during import - abandoning.", exceptionCtr);
Console.WriteLine(importNote);
LoggingHelper.DoTrace(1, importNote);
LoggingHelper.LogError(importNote, true, thisClassName + "- many exceptions");
isComplete = true;
break;
}
}
finally
{
if (!importSuccessfull)
{
if (string.IsNullOrWhiteSpace(importError))
{
importError = string.Join("\r\n", status.GetAllMessages().ToArray());
}
}
//store document
//add indicator of success
newImportId = importMgr.Add(item, entityTypeId, status.Ctid, importSuccessfull, importError, ref messages);
if (newImportId > 0 && status.Messages != null && status.Messages.Count > 0)
{
importMgr.AddMessages(newImportId, status, ref messages);
}
} //finally
if (maxRecords > 0 && cntr > maxRecords)
{
break;
}
} //end foreach
pageNbr++;
if ((maxRecords > 0 && cntr > maxRecords) || cntr == pTotalRows)
{
isComplete = true;
if (cntr < pTotalRows)
{
LoggingHelper.DoTrace(2, string.Format("CompetencyFramesworkImport EARLY EXIT. Completed {0} records out of a total of {1} ", cntr, pTotalRows));
}
}
} //
importResults = string.Format("{0} - Processed {1} records, with {2} exceptions. \r\n", thisClassName, cntr, exceptionCtr);
if (!string.IsNullOrWhiteSpace(importNote))
{
importResults += importNote;
}
//always call, as deletes are not tracked
if (UtilityManager.GetAppKeyValue("updateCompetencyFrameworkReportTotals", false) == true)
{
mgr.UpdateCompetencyFrameworkReportTotals();
}
return(importResults);
}
} //
public bool Import(string payload, string envelopeIdentifier, SaveStatus status)
{
LoggingHelper.DoTrace(7, "ImportCompetencyFramesworks - entered.");
List <string> messages = new List <string>();
MappingHelperV3 helper = new MappingHelperV3(10);
bool importSuccessfull = true;
InputEntity input = new InputEntity();
InputCompetency comp = new InputCompetency();
var mainEntity = new Dictionary <string, object>();
List <InputCompetency> competencies = new List <InputCompetency>();
Dictionary <string, object> dictionary = RegistryServices.JsonToDictionary(payload);
object graph = dictionary["@graph"];
//serialize the graph object
var glist = JsonConvert.SerializeObject(graph);
//parse graph in to list of objects
JArray graphList = JArray.Parse(glist);
var bnodes = new List <BNode>();
int cntr = 0;
foreach (var item in graphList)
{
cntr++;
//note older frameworks will not be in the priority order
var main = item.ToString();
if (cntr == 1 || main.IndexOf("ceasn:CompetencyFramework") > -1)
{
//HACK
if (main.IndexOf("ceasn:CompetencyFramework") > -1)
{
input = JsonConvert.DeserializeObject <InputEntity>(main);
}
}
else
{
//Error converting value "https://credentialengineregistry.org/resources/ce-949fcaba-45ed-44d9-88bf-43677277eb84" to type 'System.Collections.Generic.List`1[System.String]'. Path 'ceasn:isPartOf', line 11, position 108.
//not set up to handle issues
//comp = JsonConvert.DeserializeObject<InputCompetency>( item.ToString() );
//competencies.Add( comp );
//should just have competencies, but should check for bnodes
var child = item.ToString();
if (child.IndexOf("_:") > -1)
{
bnodes.Add(JsonConvert.DeserializeObject <BNode>(child));
//ceasn:Competency
}
else if (child.IndexOf("ceasn:Competency") > -1)
{
competencies.Add(JsonConvert.DeserializeObject <InputCompetency>(child));
}
else
{
//unexpected
}
}
}
//try
//{
//input = JsonConvert.DeserializeObject<InputGraph>( item.DecodedResource.ToString() );
string ctid = input.CTID;
status.Ctid = ctid;
string referencedAtId = input.CtdlId;
LoggingHelper.DoTrace(5, " ctid: "+ ctid);
LoggingHelper.DoTrace(5, " @Id: "+ input.CtdlId);
LoggingHelper.DoTrace(5, " name: "+ input.name.ToString());
string framework = input.name.ToString();
var org = new MC.Organization();
string orgCTID = "";
string orgName = "";
List <string> publisher = input.publisher;
//20-06-11 - need to get creator, publisher, owner where possible
// include an org reference with name, swp, and??
//should check creator first? Or will publisher be more likely to have an account Ctid?
if (publisher != null && publisher.Count() > 0)
{
orgCTID = ResolutionServices.ExtractCtid(publisher[0]);
//look up org name
org = OrganizationManager.GetSummaryByCtid(orgCTID);
}
else
{
//try creator
List <string> creator = input.creator;
if (creator != null && creator.Count() > 0)
{
orgCTID = ResolutionServices.ExtractCtid(creator[0]);
//look up org name
org = OrganizationManager.GetSummaryByCtid(orgCTID);
}
}
if (status.DoingDownloadOnly)
{
return(true);
}
//add/updating CompetencyFramework
//21-02-22 HUH - WHY ARE WE USING ef here instead of output
Framework ef = new Framework();
if (!DoesEntityExist(input.CTID, ref ef))
{
//set the rowid now, so that can be referenced as needed
//output.RowId = Guid.NewGuid();
ef.RowId = Guid.NewGuid();
LoggingHelper.DoTrace(1, string.Format(thisClassName + ".Import(). Record was NOT found using CTID: '{0}'", input.CTID));
}
else
{
LoggingHelper.DoTrace(1, string.Format(thisClassName + ".Import(). Found record: '{0}' using CTID: '{1}'", input.name, input.CTID));
}
helper.currentBaseObject = ef;
ef.ExistsInRegistry = true;
//store graph
ef.CompetencyFrameworkGraph = glist;
ef.TotalCompetencies = competencies.Count();
ef.Name = helper.HandleLanguageMap(input.name, ef, "Name");
ef.Description = helper.HandleLanguageMap(input.description, ef, "description");
ef.CTID = input.CTID;
ef.OrganizationCTID = orgCTID;
if (org != null && org.Id > 0)
{
orgName = org.Name;
ef.OrganizationId = org.Id;
helper.CurrentOwningAgentUid = org.RowId;
}
helper.MapInLanguageToTextValueProfile(input.inLanguage, "CompetencyFramework.InLanguage.CTID: " + ctid);
//foreach ( var l in input.InLanguage )
//{
// if ( !string.IsNullOrWhiteSpace( l ) )
// {
// var language = CodesManager.GetLanguage( l );
// output.InLanguageCodeList.Add( new TextValueProfile()
// {
// CodeId = language.CodeId,
// TextTitle = language.Name,
// TextValue = language.Value
// } );
// }
//}
//TBD handling of referencing third party publisher
if (!string.IsNullOrWhiteSpace(status.DocumentPublishedBy))
{
//output.PublishedByOrganizationCTID = status.DocumentPublishedBy;
var porg = OrganizationManager.GetSummaryByCtid(status.DocumentPublishedBy);
if (porg != null && porg.Id > 0)
{
//TODO - store this in a json blob??????????
//this will result in being added to Entity.AgentRelationship
ef.PublishedBy = new List <Guid>()
{
porg.RowId
};
}
else
{
//if publisher not imported yet, all publishee stuff will be orphaned
var entityUid = Guid.NewGuid();
var statusMsg = "";
var resPos = referencedAtId.IndexOf("/resources/");
var swp = referencedAtId.Substring(0, (resPos + "/resources/".Length)) + status.DocumentPublishedBy;
int orgId = new OrganizationManager().AddPendingRecord(entityUid, status.DocumentPublishedBy, swp, ref statusMsg);
}
}
else
{
//may need a check for existing published by to ensure not lost
if (ef.Id > 0)
{
//if ( ef.OrganizationRole != null && ef.OrganizationRole.Any() )
//{
// var publishedByList = ef.OrganizationRole.Where( s => s.RoleTypeId == 30 ).ToList();
// if ( publishedByList != null && publishedByList.Any() )
// {
// var pby = publishedByList[ 0 ].ActingAgentUid;
// ef.PublishedBy = new List<Guid>() { publishedByList[ 0 ].ActingAgentUid };
// }
//}
}
}
ef.CredentialRegistryId = envelopeIdentifier;
//additions
//ef.ind
//can only handle one source
int pcnt = 0;
if (input.source != null)
{
foreach (var url in input.source)
{
pcnt++;
ef.SourceUrl = url;
break;
}
}
ef.FrameworkUri = input.CtdlId;
ef.hasTopChild = input.hasTopChild;
//
ApiEntity apiFramework = new ApiEntity()
{
Name = ef.Name,
CTID = ef.CTID,
Source = ef.SourceUrl,
HasTopChild = ef.hasTopChild,
Meta_LastUpdated = status.LocalUpdatedDate
};
//?store competencies in string?
if (competencies != null && competencies.Count > 0)
{
ef.TotalCompetencies = competencies.Count();
//TODO - should we limit this if 1000+
//do we use competencies in elastic? if not pause this usage
cntr = 0;
foreach (var c in competencies)
{
cntr++;
var comments = helper.HandleLanguageMapList(c.comment, ef);
ef.Competencies.Add(
new workIT.Models.Elastic.IndexCompetency()
{
Name = c.competencyText.ToString(),
//CTID = c.CTID,
//Description = comments != null && comments.Count() > 0 ? comments[0].ToString() : ""
}
);
if (cntr >= 1000)
{
break;
}
}
}
//20-07-02 just storing the index ready competencies
//ef.CompentenciesJson = JsonConvert.SerializeObject( competencies, MappingHelperV3.GetJsonSettings() );
ef.CompentenciesStore = JsonConvert.SerializeObject(ef.Competencies, MappingHelperV3.GetJsonSettings());
//test
//ElasticManager.LoadCompetencies( ef.Name, ef.CompentenciesStore );
FormatCompetenciesHierarchy(apiFramework, competencies, helper);
//TODO store whole framework or just the competencies?
apiFramework.HasTopChild = null;
ef.APIFramework = JsonConvert.SerializeObject(apiFramework, MappingHelperV3.GetJsonSettings());
//
if (ef.TotalCompetencies == input.hasTopChild.Count())
{
//flat list - use for simple display
}
else
{
foreach (var item in input.hasTopChild)
{
}
}
//adding using common import pattern
new CompetencyFrameworkServices().Import(ef, ref status);
status.DocumentId = ef.Id;
status.DetailPageUrl = string.Format("~/competencyframework/{0}", ef.Id);
status.DocumentRowId = ef.RowId;
//
//just in case
if (status.HasErrors)
{
importSuccessfull = false;
}
//if record was added to db, add to/or set EntityResolution as resolved
int ierId = new ImportManager().Import_EntityResolutionAdd(referencedAtId,
ctid,
CodesManager.ENTITY_TYPE_COMPETENCY_FRAMEWORK,
output.RowId,
output.Id,
(output.Id > 0),
ref messages,
output.Id > 0);
//
//framework checks
if (input.inLanguage == null || input.inLanguage.Count() == 0)
{
//document for followup
//LoggingHelper.DoTrace( 5, " Framework missing inLanguage: " + input.name.ToString() );
}
//output.Name = helper.HandleLanguageMap( input.name, output, "Name" );
//output.description = helper.HandleLanguageMap( input.description, output, "Description" );
//output.CTID = input.CTID;
return(importSuccessfull);
}
} //
public bool Import(EntityServices mgr, string payload, string envelopeIdentifier, SaveStatus status)
{
List <string> messages = new List <string>();
MappingHelperV3 helper = new MappingHelperV3();
bool importSuccessfull = true;
InputEntity input = new InputEntity();
InputCompetency comp = new InputCompetency();
var mainEntity = new Dictionary <string, object>();
List <InputCompetency> competencies = new List <InputCompetency>();
Dictionary <string, object> dictionary = RegistryServices.JsonToDictionary(payload);
object graph = dictionary["@graph"];
//serialize the graph object
var glist = JsonConvert.SerializeObject(graph);
//parse graph in to list of objects
JArray graphList = JArray.Parse(glist);
var bnodes = new List <BNode>();
int cntr = 0;
foreach (var item in graphList)
{
cntr++;
//note older frameworks will not be in the priority order
var main = item.ToString();
if (cntr == 1 || main.IndexOf("ceasn:CompetencyFramework") > -1)
{
//HACK
if (main.IndexOf("ceasn:CompetencyFramework") > -1)
{
input = JsonConvert.DeserializeObject <InputEntity>(main);
}
}
else
{
//Error converting value "https://credentialengineregistry.org/resources/ce-949fcaba-45ed-44d9-88bf-43677277eb84" to type 'System.Collections.Generic.List`1[System.String]'. Path 'ceasn:isPartOf', line 11, position 108.
//not set up to handle issues
//comp = JsonConvert.DeserializeObject<InputCompetency>( item.ToString() );
//competencies.Add( comp );
//should just have competencies, but should check for bnodes
var child = item.ToString();
if (child.IndexOf("_:") > -1)
{
bnodes.Add(JsonConvert.DeserializeObject <BNode>(child));
//ceasn:Competency
}
else if (child.IndexOf("ceasn:Competency") > -1)
{
competencies.Add(JsonConvert.DeserializeObject <InputCompetency>(child));
}
else
{
//unexpected
}
}
}
//try
//{
//input = JsonConvert.DeserializeObject<InputGraph>( item.DecodedResource.ToString() );
string ctid = input.CTID;
status.Ctid = ctid;
string referencedAtId = input.CtdlId;
LoggingHelper.DoTrace(5, " ctid: "+ ctid);
LoggingHelper.DoTrace(5, " @Id: "+ input.CtdlId);
LoggingHelper.DoTrace(5, " name: "+ input.name.ToString());
string framework = input.name.ToString();
string orgCTID = "";
string orgName = "";
List <string> publisher = input.publisher;
//should check creator first? Or will publisher be more likely to have an account Ctid?
if (publisher != null && publisher.Count() > 0)
{
orgCTID = ResolutionServices.ExtractCtid(publisher[0]);
//look up org name
orgName = OrganizationManager.GetByCtid(orgCTID).Name ?? "missing";
}
else
{
//try creator
List <string> creator = input.creator;
if (creator != null && creator.Count() > 0)
{
orgCTID = ResolutionServices.ExtractCtid(creator[0]);
//look up org name
orgName = OrganizationManager.GetByCtid(orgCTID).Name ?? "missing";
}
}
if (status.DoingDownloadOnly)
{
return(true);
}
//add updating educationFramework
Framework ef = new Framework();
if (!DoesEntityExist(input.CTID, ref ef))
{
//set the rowid now, so that can be referenced as needed
output.RowId = Guid.NewGuid();
ef.RowId = Guid.NewGuid();
}
helper.currentBaseObject = ef;
ef.ExistsInRegistry = true;
ef.FrameworkName = helper.HandleLanguageMap(input.name, ef, "Name");
ef.CTID = input.CTID;
ef.OrganizationCTID = orgCTID;
ef.CredentialRegistryId = envelopeIdentifier;
//can only handle one source
int pcnt = 0;
foreach (var url in input.source)
{
pcnt++;
ef.SourceUrl = url;
break;
}
ef.FrameworkUri = input.CtdlId;
new EducationFrameworkManager().Save(ef, ref status, true);
//
//
//framework checks
if (input.inLanguage == null || input.inLanguage.Count() == 0)
{
//document for followup
//LoggingHelper.DoTrace( 5, " Framework missing inLanguage: " + input.name.ToString() );
}
//output.Name = helper.HandleLanguageMap( input.name, output, "Name" );
//output.description = helper.HandleLanguageMap( input.description, output, "Description" );
output.Ctid = input.CTID;
return(importSuccessfull);
}
