private InputEntity GetFramework(object graph) { //string ctid = ""; InputEntity entity = new InputEntity(); Newtonsoft.Json.Linq.JArray jarray = (Newtonsoft.Json.Linq.JArray)graph; foreach (var token in jarray) { if (token.GetType() == typeof(Newtonsoft.Json.Linq.JObject)) { if (token.ToString().IndexOf("ceasn:CompetencyFramework") > -1) { entity = ((Newtonsoft.Json.Linq.JObject)token).ToObject <InputEntity>(); //InputEntity cf = ( InputEntity ) JsonConvert.DeserializeObject( token.ToString() ); return(entity); } else if (token.ToString().IndexOf("ceasn:Competency") > -1) { //ignore //var c1 = token.ToString().Replace( "exactMatch", "exactAlignment" ); //var c2 = ( ( Newtonsoft.Json.Linq.JObject ) c1 ).ToObject<RA.Models.Json.InputCompetency>(); } //var itemProperties = token.Children<JProperty>(); ////you could do a foreach or a linq here depending on what you need to do exactly with the value //var myElement = itemProperties.FirstOrDefault( x => x.Name == "url" ); //var myElementValue = myElement.Value; ////This is a JValue type } else { //error } } //no ctid found, so???? return(entity); }
public string Import(string startingDate, string endingDate, int maxRecords, string community, bool downloadOnly = false) { bool importingThisType = UtilityManager.GetAppKeyValue("importing_competency_frameworks", true); if (!importingThisType) { LoggingHelper.DoTrace(1, "=== ***************** Skipping import of Competency Frameworks ***************** "); return("Skipped import of competency_frameworks"); } LoggingHelper.DoTrace(1, string.Format("=== ***************** {0} ***************** ", thisClassName)); JsonEntity input = new JsonEntity(); ReadEnvelope envelope = new ReadEnvelope(); List <ReadEnvelope> list = new List <ReadEnvelope>(); EntityServices mgr = new EntityServices(); string type = "competency_framework"; int pageNbr = 1; int pageSize = 75; string importError = ""; string importResults = ""; string importNote = ""; ThisEntity output = new ThisEntity(); List <string> messages = new List <string>(); //the nbr of records needs to be monitored, to determine the optimum //NOTE: consider the IOER approach that all candidate records are first downloaded, and then a separate process does the import int cntr = 0; int pTotalRows = 0; int exceptionCtr = 0; string statusMessage = ""; bool isComplete = false; bool importSuccessfull = true; int newImportId = 0; SaveStatus status = new SaveStatus(); //will need to handle multiple calls - watch for time outs while (pageNbr > 0 && !isComplete) { list = RegistryServices.Search(type, startingDate, endingDate, pageNbr, pageSize, ref pTotalRows, ref statusMessage, community); //list = GetLatest( type, startingDate, endingDate, pageNbr, pageSize, ref pTotalRows, ref statusMessage, community ); if (list == null || list.Count == 0) { isComplete = true; if (pageNbr == 1) { importNote = "Competency Frameworks: No records where found for date range "; //Console.WriteLine( thisClassName + importNote ); LoggingHelper.DoTrace(4, thisClassName + importNote); } break; } foreach (ReadEnvelope item in list) { cntr++; string payload = item.DecodedResource.ToString(); LoggingHelper.DoTrace(2, string.Format("{0}. EnvelopeIdentifier: {1} ", cntr, item.EnvelopeIdentifier)); status = new SaveStatus(); status.DoingDownloadOnly = downloadOnly; importError = ""; importSuccessfull = false; try { //Console.WriteLine( string.Format( "{0}. Competency Frameswork Envelope Identifier {1} ", cntr, item.EnvelopeIdentifier ) ); importSuccessfull = entityImportMgr.ProcessEnvelope(mgr, item, status); } catch (Exception ex) { if (ex.Message.IndexOf("Path '@context', line 1") > 0) { importError = "The referenced registry document is using an old schema. Please republish it with the latest schema!"; status.AddError(importError); } else { LoggingHelper.LogError(ex, string.Format("Exception encountered in envelopeId: {0}", item.EnvelopeIdentifier), false, "CredentialFinder Import exception"); status.AddError(ex.Message); importError = ex.Message; LoggingHelper.DoTrace(1, " show exception to use getting related envelopeId " + ex.Message); } //make continue on exceptions an option exceptionCtr++; if (maxExceptions > 0 && exceptionCtr > maxExceptions) { //arbitrarily stop if large number of exceptions importNote = string.Format(thisClassName + " - Many exceptions ({0}) were encountered during import - abandoning.", exceptionCtr); Console.WriteLine(importNote); LoggingHelper.DoTrace(1, importNote); LoggingHelper.LogError(importNote, true, thisClassName + "- many exceptions"); isComplete = true; break; } } finally { if (!importSuccessfull) { if (string.IsNullOrWhiteSpace(importError)) { importError = string.Join("\r\n", status.GetAllMessages().ToArray()); } } //store document //add indicator of success newImportId = importMgr.Add(item, entityTypeId, status.Ctid, importSuccessfull, importError, ref messages); if (newImportId > 0 && status.Messages != null && status.Messages.Count > 0) { importMgr.AddMessages(newImportId, status, ref messages); } } //finally if (maxRecords > 0 && cntr > maxRecords) { break; } } //end foreach pageNbr++; if ((maxRecords > 0 && cntr > maxRecords) || cntr == pTotalRows) { isComplete = true; if (cntr < pTotalRows) { LoggingHelper.DoTrace(2, string.Format("CompetencyFramesworkImport EARLY EXIT. Completed {0} records out of a total of {1} ", cntr, pTotalRows)); } } } // importResults = string.Format("{0} - Processed {1} records, with {2} exceptions. \r\n", thisClassName, cntr, exceptionCtr); if (!string.IsNullOrWhiteSpace(importNote)) { importResults += importNote; } //always call, as deletes are not tracked if (UtilityManager.GetAppKeyValue("updateCompetencyFrameworkReportTotals", false) == true) { mgr.UpdateCompetencyFrameworkReportTotals(); } return(importResults); }
} // public bool Import(string payload, string envelopeIdentifier, SaveStatus status) { LoggingHelper.DoTrace(7, "ImportCompetencyFramesworks - entered."); List <string> messages = new List <string>(); MappingHelperV3 helper = new MappingHelperV3(10); bool importSuccessfull = true; InputEntity input = new InputEntity(); InputCompetency comp = new InputCompetency(); var mainEntity = new Dictionary <string, object>(); List <InputCompetency> competencies = new List <InputCompetency>(); Dictionary <string, object> dictionary = RegistryServices.JsonToDictionary(payload); object graph = dictionary["@graph"]; //serialize the graph object var glist = JsonConvert.SerializeObject(graph); //parse graph in to list of objects JArray graphList = JArray.Parse(glist); var bnodes = new List <BNode>(); int cntr = 0; foreach (var item in graphList) { cntr++; //note older frameworks will not be in the priority order var main = item.ToString(); if (cntr == 1 || main.IndexOf("ceasn:CompetencyFramework") > -1) { //HACK if (main.IndexOf("ceasn:CompetencyFramework") > -1) { input = JsonConvert.DeserializeObject <InputEntity>(main); } } else { //Error converting value "https://credentialengineregistry.org/resources/ce-949fcaba-45ed-44d9-88bf-43677277eb84" to type 'System.Collections.Generic.List`1[System.String]'. Path 'ceasn:isPartOf', line 11, position 108. //not set up to handle issues //comp = JsonConvert.DeserializeObject<InputCompetency>( item.ToString() ); //competencies.Add( comp ); //should just have competencies, but should check for bnodes var child = item.ToString(); if (child.IndexOf("_:") > -1) { bnodes.Add(JsonConvert.DeserializeObject <BNode>(child)); //ceasn:Competency } else if (child.IndexOf("ceasn:Competency") > -1) { competencies.Add(JsonConvert.DeserializeObject <InputCompetency>(child)); } else { //unexpected } } } //try //{ //input = JsonConvert.DeserializeObject<InputGraph>( item.DecodedResource.ToString() ); string ctid = input.CTID; status.Ctid = ctid; string referencedAtId = input.CtdlId; LoggingHelper.DoTrace(5, " ctid: "+ ctid); LoggingHelper.DoTrace(5, " @Id: "+ input.CtdlId); LoggingHelper.DoTrace(5, " name: "+ input.name.ToString()); string framework = input.name.ToString(); var org = new MC.Organization(); string orgCTID = ""; string orgName = ""; List <string> publisher = input.publisher; //20-06-11 - need to get creator, publisher, owner where possible // include an org reference with name, swp, and?? //should check creator first? Or will publisher be more likely to have an account Ctid? if (publisher != null && publisher.Count() > 0) { orgCTID = ResolutionServices.ExtractCtid(publisher[0]); //look up org name org = OrganizationManager.GetSummaryByCtid(orgCTID); } else { //try creator List <string> creator = input.creator; if (creator != null && creator.Count() > 0) { orgCTID = ResolutionServices.ExtractCtid(creator[0]); //look up org name org = OrganizationManager.GetSummaryByCtid(orgCTID); } } if (status.DoingDownloadOnly) { return(true); } //add/updating CompetencyFramework //21-02-22 HUH - WHY ARE WE USING ef here instead of output Framework ef = new Framework(); if (!DoesEntityExist(input.CTID, ref ef)) { //set the rowid now, so that can be referenced as needed //output.RowId = Guid.NewGuid(); ef.RowId = Guid.NewGuid(); LoggingHelper.DoTrace(1, string.Format(thisClassName + ".Import(). Record was NOT found using CTID: '{0}'", input.CTID)); } else { LoggingHelper.DoTrace(1, string.Format(thisClassName + ".Import(). Found record: '{0}' using CTID: '{1}'", input.name, input.CTID)); } helper.currentBaseObject = ef; ef.ExistsInRegistry = true; //store graph ef.CompetencyFrameworkGraph = glist; ef.TotalCompetencies = competencies.Count(); ef.Name = helper.HandleLanguageMap(input.name, ef, "Name"); ef.Description = helper.HandleLanguageMap(input.description, ef, "description"); ef.CTID = input.CTID; ef.OrganizationCTID = orgCTID; if (org != null && org.Id > 0) { orgName = org.Name; ef.OrganizationId = org.Id; helper.CurrentOwningAgentUid = org.RowId; } helper.MapInLanguageToTextValueProfile(input.inLanguage, "CompetencyFramework.InLanguage.CTID: " + ctid); //foreach ( var l in input.InLanguage ) //{ // if ( !string.IsNullOrWhiteSpace( l ) ) // { // var language = CodesManager.GetLanguage( l ); // output.InLanguageCodeList.Add( new TextValueProfile() // { // CodeId = language.CodeId, // TextTitle = language.Name, // TextValue = language.Value // } ); // } //} //TBD handling of referencing third party publisher if (!string.IsNullOrWhiteSpace(status.DocumentPublishedBy)) { //output.PublishedByOrganizationCTID = status.DocumentPublishedBy; var porg = OrganizationManager.GetSummaryByCtid(status.DocumentPublishedBy); if (porg != null && porg.Id > 0) { //TODO - store this in a json blob?????????? //this will result in being added to Entity.AgentRelationship ef.PublishedBy = new List <Guid>() { porg.RowId }; } else { //if publisher not imported yet, all publishee stuff will be orphaned var entityUid = Guid.NewGuid(); var statusMsg = ""; var resPos = referencedAtId.IndexOf("/resources/"); var swp = referencedAtId.Substring(0, (resPos + "/resources/".Length)) + status.DocumentPublishedBy; int orgId = new OrganizationManager().AddPendingRecord(entityUid, status.DocumentPublishedBy, swp, ref statusMsg); } } else { //may need a check for existing published by to ensure not lost if (ef.Id > 0) { //if ( ef.OrganizationRole != null && ef.OrganizationRole.Any() ) //{ // var publishedByList = ef.OrganizationRole.Where( s => s.RoleTypeId == 30 ).ToList(); // if ( publishedByList != null && publishedByList.Any() ) // { // var pby = publishedByList[ 0 ].ActingAgentUid; // ef.PublishedBy = new List<Guid>() { publishedByList[ 0 ].ActingAgentUid }; // } //} } } ef.CredentialRegistryId = envelopeIdentifier; //additions //ef.ind //can only handle one source int pcnt = 0; if (input.source != null) { foreach (var url in input.source) { pcnt++; ef.SourceUrl = url; break; } } ef.FrameworkUri = input.CtdlId; ef.hasTopChild = input.hasTopChild; // ApiEntity apiFramework = new ApiEntity() { Name = ef.Name, CTID = ef.CTID, Source = ef.SourceUrl, HasTopChild = ef.hasTopChild, Meta_LastUpdated = status.LocalUpdatedDate }; //?store competencies in string? if (competencies != null && competencies.Count > 0) { ef.TotalCompetencies = competencies.Count(); //TODO - should we limit this if 1000+ //do we use competencies in elastic? if not pause this usage cntr = 0; foreach (var c in competencies) { cntr++; var comments = helper.HandleLanguageMapList(c.comment, ef); ef.Competencies.Add( new workIT.Models.Elastic.IndexCompetency() { Name = c.competencyText.ToString(), //CTID = c.CTID, //Description = comments != null && comments.Count() > 0 ? comments[0].ToString() : "" } ); if (cntr >= 1000) { break; } } } //20-07-02 just storing the index ready competencies //ef.CompentenciesJson = JsonConvert.SerializeObject( competencies, MappingHelperV3.GetJsonSettings() ); ef.CompentenciesStore = JsonConvert.SerializeObject(ef.Competencies, MappingHelperV3.GetJsonSettings()); //test //ElasticManager.LoadCompetencies( ef.Name, ef.CompentenciesStore ); FormatCompetenciesHierarchy(apiFramework, competencies, helper); //TODO store whole framework or just the competencies? apiFramework.HasTopChild = null; ef.APIFramework = JsonConvert.SerializeObject(apiFramework, MappingHelperV3.GetJsonSettings()); // if (ef.TotalCompetencies == input.hasTopChild.Count()) { //flat list - use for simple display } else { foreach (var item in input.hasTopChild) { } } //adding using common import pattern new CompetencyFrameworkServices().Import(ef, ref status); status.DocumentId = ef.Id; status.DetailPageUrl = string.Format("~/competencyframework/{0}", ef.Id); status.DocumentRowId = ef.RowId; // //just in case if (status.HasErrors) { importSuccessfull = false; } //if record was added to db, add to/or set EntityResolution as resolved int ierId = new ImportManager().Import_EntityResolutionAdd(referencedAtId, ctid, CodesManager.ENTITY_TYPE_COMPETENCY_FRAMEWORK, output.RowId, output.Id, (output.Id > 0), ref messages, output.Id > 0); // //framework checks if (input.inLanguage == null || input.inLanguage.Count() == 0) { //document for followup //LoggingHelper.DoTrace( 5, " Framework missing inLanguage: " + input.name.ToString() ); } //output.Name = helper.HandleLanguageMap( input.name, output, "Name" ); //output.description = helper.HandleLanguageMap( input.description, output, "Description" ); //output.CTID = input.CTID; return(importSuccessfull); }
} // public bool Import(EntityServices mgr, string payload, string envelopeIdentifier, SaveStatus status) { List <string> messages = new List <string>(); MappingHelperV3 helper = new MappingHelperV3(); bool importSuccessfull = true; InputEntity input = new InputEntity(); InputCompetency comp = new InputCompetency(); var mainEntity = new Dictionary <string, object>(); List <InputCompetency> competencies = new List <InputCompetency>(); Dictionary <string, object> dictionary = RegistryServices.JsonToDictionary(payload); object graph = dictionary["@graph"]; //serialize the graph object var glist = JsonConvert.SerializeObject(graph); //parse graph in to list of objects JArray graphList = JArray.Parse(glist); var bnodes = new List <BNode>(); int cntr = 0; foreach (var item in graphList) { cntr++; //note older frameworks will not be in the priority order var main = item.ToString(); if (cntr == 1 || main.IndexOf("ceasn:CompetencyFramework") > -1) { //HACK if (main.IndexOf("ceasn:CompetencyFramework") > -1) { input = JsonConvert.DeserializeObject <InputEntity>(main); } } else { //Error converting value "https://credentialengineregistry.org/resources/ce-949fcaba-45ed-44d9-88bf-43677277eb84" to type 'System.Collections.Generic.List`1[System.String]'. Path 'ceasn:isPartOf', line 11, position 108. //not set up to handle issues //comp = JsonConvert.DeserializeObject<InputCompetency>( item.ToString() ); //competencies.Add( comp ); //should just have competencies, but should check for bnodes var child = item.ToString(); if (child.IndexOf("_:") > -1) { bnodes.Add(JsonConvert.DeserializeObject <BNode>(child)); //ceasn:Competency } else if (child.IndexOf("ceasn:Competency") > -1) { competencies.Add(JsonConvert.DeserializeObject <InputCompetency>(child)); } else { //unexpected } } } //try //{ //input = JsonConvert.DeserializeObject<InputGraph>( item.DecodedResource.ToString() ); string ctid = input.CTID; status.Ctid = ctid; string referencedAtId = input.CtdlId; LoggingHelper.DoTrace(5, " ctid: "+ ctid); LoggingHelper.DoTrace(5, " @Id: "+ input.CtdlId); LoggingHelper.DoTrace(5, " name: "+ input.name.ToString()); string framework = input.name.ToString(); string orgCTID = ""; string orgName = ""; List <string> publisher = input.publisher; //should check creator first? Or will publisher be more likely to have an account Ctid? if (publisher != null && publisher.Count() > 0) { orgCTID = ResolutionServices.ExtractCtid(publisher[0]); //look up org name orgName = OrganizationManager.GetByCtid(orgCTID).Name ?? "missing"; } else { //try creator List <string> creator = input.creator; if (creator != null && creator.Count() > 0) { orgCTID = ResolutionServices.ExtractCtid(creator[0]); //look up org name orgName = OrganizationManager.GetByCtid(orgCTID).Name ?? "missing"; } } if (status.DoingDownloadOnly) { return(true); } //add updating educationFramework Framework ef = new Framework(); if (!DoesEntityExist(input.CTID, ref ef)) { //set the rowid now, so that can be referenced as needed output.RowId = Guid.NewGuid(); ef.RowId = Guid.NewGuid(); } helper.currentBaseObject = ef; ef.ExistsInRegistry = true; ef.FrameworkName = helper.HandleLanguageMap(input.name, ef, "Name"); ef.CTID = input.CTID; ef.OrganizationCTID = orgCTID; ef.CredentialRegistryId = envelopeIdentifier; //can only handle one source int pcnt = 0; foreach (var url in input.source) { pcnt++; ef.SourceUrl = url; break; } ef.FrameworkUri = input.CtdlId; new EducationFrameworkManager().Save(ef, ref status, true); // // //framework checks if (input.inLanguage == null || input.inLanguage.Count() == 0) { //document for followup //LoggingHelper.DoTrace( 5, " Framework missing inLanguage: " + input.name.ToString() ); } //output.Name = helper.HandleLanguageMap( input.name, output, "Name" ); //output.description = helper.HandleLanguageMap( input.description, output, "Description" ); output.Ctid = input.CTID; return(importSuccessfull); }