} // public bool Import(string payload, string envelopeIdentifier, SaveStatus status) { LoggingHelper.DoTrace(7, "ImportCompetencyFramesworks - entered."); List <string> messages = new List <string>(); MappingHelperV3 helper = new MappingHelperV3(10); bool importSuccessfull = true; InputEntity input = new InputEntity(); InputCompetency comp = new InputCompetency(); var mainEntity = new Dictionary <string, object>(); List <InputCompetency> competencies = new List <InputCompetency>(); Dictionary <string, object> dictionary = RegistryServices.JsonToDictionary(payload); object graph = dictionary["@graph"]; //serialize the graph object var glist = JsonConvert.SerializeObject(graph); //parse graph in to list of objects JArray graphList = JArray.Parse(glist); var bnodes = new List <BNode>(); int cntr = 0; foreach (var item in graphList) { cntr++; //note older frameworks will not be in the priority order var main = item.ToString(); if (cntr == 1 || main.IndexOf("ceasn:CompetencyFramework") > -1) { //HACK if (main.IndexOf("ceasn:CompetencyFramework") > -1) { input = JsonConvert.DeserializeObject <InputEntity>(main); } } else { //Error converting value "https://credentialengineregistry.org/resources/ce-949fcaba-45ed-44d9-88bf-43677277eb84" to type 'System.Collections.Generic.List`1[System.String]'. Path 'ceasn:isPartOf', line 11, position 108. //not set up to handle issues //comp = JsonConvert.DeserializeObject<InputCompetency>( item.ToString() ); //competencies.Add( comp ); //should just have competencies, but should check for bnodes var child = item.ToString(); if (child.IndexOf("_:") > -1) { bnodes.Add(JsonConvert.DeserializeObject <BNode>(child)); //ceasn:Competency } else if (child.IndexOf("ceasn:Competency") > -1) { competencies.Add(JsonConvert.DeserializeObject <InputCompetency>(child)); } else { //unexpected } } } //try //{ //input = JsonConvert.DeserializeObject<InputGraph>( item.DecodedResource.ToString() ); string ctid = input.CTID; status.Ctid = ctid; string referencedAtId = input.CtdlId; LoggingHelper.DoTrace(5, " ctid: "+ ctid); LoggingHelper.DoTrace(5, " @Id: "+ input.CtdlId); LoggingHelper.DoTrace(5, " name: "+ input.name.ToString()); string framework = input.name.ToString(); var org = new MC.Organization(); string orgCTID = ""; string orgName = ""; List <string> publisher = input.publisher; //20-06-11 - need to get creator, publisher, owner where possible // include an org reference with name, swp, and?? //should check creator first? Or will publisher be more likely to have an account Ctid? if (publisher != null && publisher.Count() > 0) { orgCTID = ResolutionServices.ExtractCtid(publisher[0]); //look up org name org = OrganizationManager.GetSummaryByCtid(orgCTID); } else { //try creator List <string> creator = input.creator; if (creator != null && creator.Count() > 0) { orgCTID = ResolutionServices.ExtractCtid(creator[0]); //look up org name org = OrganizationManager.GetSummaryByCtid(orgCTID); } } if (status.DoingDownloadOnly) { return(true); } //add/updating CompetencyFramework //21-02-22 HUH - WHY ARE WE USING ef here instead of output Framework ef = new Framework(); if (!DoesEntityExist(input.CTID, ref ef)) { //set the rowid now, so that can be referenced as needed //output.RowId = Guid.NewGuid(); ef.RowId = Guid.NewGuid(); LoggingHelper.DoTrace(1, string.Format(thisClassName + ".Import(). Record was NOT found using CTID: '{0}'", input.CTID)); } else { LoggingHelper.DoTrace(1, string.Format(thisClassName + ".Import(). Found record: '{0}' using CTID: '{1}'", input.name, input.CTID)); } helper.currentBaseObject = ef; ef.ExistsInRegistry = true; //store graph ef.CompetencyFrameworkGraph = glist; ef.TotalCompetencies = competencies.Count(); ef.Name = helper.HandleLanguageMap(input.name, ef, "Name"); ef.Description = helper.HandleLanguageMap(input.description, ef, "description"); ef.CTID = input.CTID; ef.OrganizationCTID = orgCTID; if (org != null && org.Id > 0) { orgName = org.Name; ef.OrganizationId = org.Id; helper.CurrentOwningAgentUid = org.RowId; } helper.MapInLanguageToTextValueProfile(input.inLanguage, "CompetencyFramework.InLanguage.CTID: " + ctid); //foreach ( var l in input.InLanguage ) //{ // if ( !string.IsNullOrWhiteSpace( l ) ) // { // var language = CodesManager.GetLanguage( l ); // output.InLanguageCodeList.Add( new TextValueProfile() // { // CodeId = language.CodeId, // TextTitle = language.Name, // TextValue = language.Value // } ); // } //} //TBD handling of referencing third party publisher if (!string.IsNullOrWhiteSpace(status.DocumentPublishedBy)) { //output.PublishedByOrganizationCTID = status.DocumentPublishedBy; var porg = OrganizationManager.GetSummaryByCtid(status.DocumentPublishedBy); if (porg != null && porg.Id > 0) { //TODO - store this in a json blob?????????? //this will result in being added to Entity.AgentRelationship ef.PublishedBy = new List <Guid>() { porg.RowId }; } else { //if publisher not imported yet, all publishee stuff will be orphaned var entityUid = Guid.NewGuid(); var statusMsg = ""; var resPos = referencedAtId.IndexOf("/resources/"); var swp = referencedAtId.Substring(0, (resPos + "/resources/".Length)) + status.DocumentPublishedBy; int orgId = new OrganizationManager().AddPendingRecord(entityUid, status.DocumentPublishedBy, swp, ref statusMsg); } } else { //may need a check for existing published by to ensure not lost if (ef.Id > 0) { //if ( ef.OrganizationRole != null && ef.OrganizationRole.Any() ) //{ // var publishedByList = ef.OrganizationRole.Where( s => s.RoleTypeId == 30 ).ToList(); // if ( publishedByList != null && publishedByList.Any() ) // { // var pby = publishedByList[ 0 ].ActingAgentUid; // ef.PublishedBy = new List<Guid>() { publishedByList[ 0 ].ActingAgentUid }; // } //} } } ef.CredentialRegistryId = envelopeIdentifier; //additions //ef.ind //can only handle one source int pcnt = 0; if (input.source != null) { foreach (var url in input.source) { pcnt++; ef.SourceUrl = url; break; } } ef.FrameworkUri = input.CtdlId; ef.hasTopChild = input.hasTopChild; // ApiEntity apiFramework = new ApiEntity() { Name = ef.Name, CTID = ef.CTID, Source = ef.SourceUrl, HasTopChild = ef.hasTopChild, Meta_LastUpdated = status.LocalUpdatedDate }; //?store competencies in string? if (competencies != null && competencies.Count > 0) { ef.TotalCompetencies = competencies.Count(); //TODO - should we limit this if 1000+ //do we use competencies in elastic? if not pause this usage cntr = 0; foreach (var c in competencies) { cntr++; var comments = helper.HandleLanguageMapList(c.comment, ef); ef.Competencies.Add( new workIT.Models.Elastic.IndexCompetency() { Name = c.competencyText.ToString(), //CTID = c.CTID, //Description = comments != null && comments.Count() > 0 ? comments[0].ToString() : "" } ); if (cntr >= 1000) { break; } } } //20-07-02 just storing the index ready competencies //ef.CompentenciesJson = JsonConvert.SerializeObject( competencies, MappingHelperV3.GetJsonSettings() ); ef.CompentenciesStore = JsonConvert.SerializeObject(ef.Competencies, MappingHelperV3.GetJsonSettings()); //test //ElasticManager.LoadCompetencies( ef.Name, ef.CompentenciesStore ); FormatCompetenciesHierarchy(apiFramework, competencies, helper); //TODO store whole framework or just the competencies? apiFramework.HasTopChild = null; ef.APIFramework = JsonConvert.SerializeObject(apiFramework, MappingHelperV3.GetJsonSettings()); // if (ef.TotalCompetencies == input.hasTopChild.Count()) { //flat list - use for simple display } else { foreach (var item in input.hasTopChild) { } } //adding using common import pattern new CompetencyFrameworkServices().Import(ef, ref status); status.DocumentId = ef.Id; status.DetailPageUrl = string.Format("~/competencyframework/{0}", ef.Id); status.DocumentRowId = ef.RowId; // //just in case if (status.HasErrors) { importSuccessfull = false; } //if record was added to db, add to/or set EntityResolution as resolved int ierId = new ImportManager().Import_EntityResolutionAdd(referencedAtId, ctid, CodesManager.ENTITY_TYPE_COMPETENCY_FRAMEWORK, output.RowId, output.Id, (output.Id > 0), ref messages, output.Id > 0); // //framework checks if (input.inLanguage == null || input.inLanguage.Count() == 0) { //document for followup //LoggingHelper.DoTrace( 5, " Framework missing inLanguage: " + input.name.ToString() ); } //output.Name = helper.HandleLanguageMap( input.name, output, "Name" ); //output.description = helper.HandleLanguageMap( input.description, output, "Description" ); //output.CTID = input.CTID; return(importSuccessfull); }
} // public bool Import(string payload, string envelopeIdentifier, SaveStatus status) { /* checklist * * Y ceasn:abilityEmbodied * Y ceasn:comment * Y ceasn:knowledgeEmbodied * Y ceasn:skillEmbodied * Y ceterms:classification * Y ceterms:ctid * Y ceterms:description * ceterms:hasJob - to Job * ceterms:hasSpecialization - to narrower Occupation * ceterms:hasWorkRole - to WorkRole * Y ceterms:identifier * Y ceterms:industryType * ceterms:isSpecializationOf - to broader occupation * Y ceterms:keyword * Y ceterms:name * Y ceterms:occupationType * Y ceterms:sameAs * Y ceterms:subjectWebpage * Y ceterms:versionIdentifier * */ LoggingHelper.DoTrace(6, "ImportOccupations - entered."); List <string> messages = new List <string>(); bool importSuccessfull = false; EntityServices mgr = new EntityServices(); // InputEntity input = new InputEntity(); var mainEntity = new Dictionary <string, object>(); // Dictionary <string, object> dictionary = RegistryServices.JsonToDictionary(payload); object graph = dictionary["@graph"]; //serialize the graph object var glist = JsonConvert.SerializeObject(graph); //parse graph in to list of objects JArray graphList = JArray.Parse(glist); var bnodes = new List <BNode>(); int cntr = 0; foreach (var item in graphList) { cntr++; if (cntr == 1) { var main = item.ToString(); //may not use this. Could add a trace method mainEntity = RegistryServices.JsonToDictionary(main); input = JsonConvert.DeserializeObject <InputEntity>(main); } else { //may have blank nodes? var bn = item.ToString(); bnodes.Add(JsonConvert.DeserializeObject <BNode>(bn)); var child = item.ToString(); if (child.IndexOf("_:") > -1) { bnodes.Add(JsonConvert.DeserializeObject <BNode>(child)); } else { //unexpected Dictionary <string, object> unexpected = RegistryServices.JsonToDictionary(child); object unexpectedType = unexpected["@type"]; status.AddError("Unexpected document type"); } } } MappingHelperV3 helper = new MappingHelperV3(entityTypeId); helper.entityBlankNodes = bnodes; helper.CurrentEntityCTID = input.CTID; helper.CurrentEntityName = input.Name.ToString(); status.EnvelopeId = envelopeIdentifier; try { string ctid = input.CTID; string referencedAtId = input.CtdlId; LoggingHelper.DoTrace(5, " name: "+ input.Name.ToString()); LoggingHelper.DoTrace(6, " url: "+ input.SubjectWebpage); LoggingHelper.DoTrace(5, " ctid: "+ input.CTID); LoggingHelper.DoTrace(6, " @Id: "+ input.CtdlId); status.Ctid = ctid; if (status.DoingDownloadOnly) { return(true); } if (!DoesEntityExist(input.CTID, ref output)) { //set the rowid now, so that can be referenced as needed output.RowId = Guid.NewGuid(); LoggingHelper.DoTrace(1, string.Format(thisClassName + ".Import(). Record was NOT found using CTID: '{0}'", input.CTID)); } else { LoggingHelper.DoTrace(1, string.Format(thisClassName + ".Import(). Found record: '{0}' using CTID: '{1}'", input.Name, input.CTID)); } helper.currentBaseObject = output; output.Name = helper.HandleLanguageMap(input.Name, output, "Name"); output.Description = helper.HandleLanguageMap(input.Description, output, "Description"); output.SubjectWebpage = input.SubjectWebpage; output.CTID = input.CTID; //TBD handling of referencing third party publisher if (!string.IsNullOrWhiteSpace(status.DocumentPublishedBy)) { //output.PublishedByOrganizationCTID = status.DocumentPublishedBy; var porg = OrganizationManager.GetSummaryByCtid(status.DocumentPublishedBy); if (porg != null && porg.Id > 0) { //TODO - store this in a json blob?????????? //this will result in being added to Entity.AgentRelationship output.PublishedBy = new List <Guid>() { porg.RowId }; } else { //if publisher not imported yet, all publishee stuff will be orphaned var entityUid = Guid.NewGuid(); var statusMsg = ""; var resPos = referencedAtId.IndexOf("/resources/"); var swp = referencedAtId.Substring(0, (resPos + "/resources/".Length)) + status.DocumentPublishedBy; int orgId = new OrganizationManager().AddPendingRecord(entityUid, status.DocumentPublishedBy, swp, ref statusMsg); } } else { //may need a check for existing published by to ensure not lost //????????????? if (output.Id > 0) { //if ( output.OrganizationRole != null && output.OrganizationRole.Any() ) //{ // var publishedByList = output.OrganizationRole.Where( s => s.RoleTypeId == 30 ).ToList(); // if ( publishedByList != null && publishedByList.Any() ) // { // var pby = publishedByList[ 0 ].ActingAgentUid; // output.PublishedBy = new List<Guid>() { publishedByList[ 0 ].ActingAgentUid }; // } //} } } //warning this gets set to blank if doing a manual import by ctid output.CredentialRegistryId = envelopeIdentifier; output.Identifier = helper.MapIdentifierValueListInternal(input.Identifier); if (output.Identifier != null && output.Identifier.Count() > 0) { output.IdentifierJson = JsonConvert.SerializeObject(output.Identifier, MappingHelperV3.GetJsonSettings()); } // output.Comment = helper.HandleLanguageMapList(input.Comment, output); //what to do with these uris? output.AbilityEmbodied = input.AbilityEmbodied; //a concept, but unknown concept scheme? output.Classification = input.Classification; output.KnowledgeEmbodied = input.KnowledgeEmbodied; output.SkillEmbodied = input.SkillEmbodied; //SameAs URI - need to chg this to json - output.SameAs = helper.MapToTextValueProfile(input.SameAs); // output.VersionIdentifier = helper.MapIdentifierValueListInternal(input.VersionIdentifier); if (output.VersionIdentifier != null && output.VersionIdentifier.Count() > 0) { output.VersionIdentifierJson = JsonConvert.SerializeObject(output.VersionIdentifier, MappingHelperV3.GetJsonSettings()); } output.Keyword = helper.MapToTextValueProfile(input.Keyword, output, "Keyword"); //Industries/occupations output.Industries = helper.MapCAOListToCAOProfileList(input.IndustryType); output.Occupations = helper.MapCAOListToCAOProfileList(input.OccupationType); //may need to save the occupation and then handle components //or do a create pending for hasDestination and any hasChild (actually already done by MapEntityReferenceGuids) //=== if any messages were encountered treat as warnings for now if (messages.Count > 0) { status.SetMessages(messages, true); } // //adding common import pattern importSuccessfull = mgr.Import(output, ref status); // status.DocumentId = output.Id; status.DetailPageUrl = string.Format("~/occupation/{0}", output.Id); status.DocumentRowId = output.RowId; //if record was added to db, add to/or set EntityResolution as resolved int ierId = new ImportManager().Import_EntityResolutionAdd(referencedAtId, ctid, entityTypeId, output.RowId, output.Id, (output.Id > 0), ref messages, output.Id > 0); //just in case - not sure if applicable, as will want to do components if the occupation exists if (status.HasErrors) { importSuccessfull = false; //email an error report, and/or add to activity log? } } catch (Exception ex) { LoggingHelper.LogError(ex, string.Format("Exception encountered in envelopeId: {0}", envelopeIdentifier), false, "Occupation Import exception"); } return(importSuccessfull); }
} // public bool Import(string payload, string envelopeIdentifier, SaveStatus status) { LoggingHelper.DoTrace(7, "ImportConceptSchemes - entered."); List <string> messages = new List <string>(); MappingHelperV3 helper = new MappingHelperV3(10); bool importSuccessfull = true; var input = new InputEntity(); var concept = new InputConcept(); var mainEntity = new Dictionary <string, object>(); List <InputConcept> concepts = new List <InputConcept>(); Dictionary <string, object> dictionary = RegistryServices.JsonToDictionary(payload); object graph = dictionary["@graph"]; //serialize the graph object var glist = JsonConvert.SerializeObject(graph); //parse graph in to list of objects JArray graphList = JArray.Parse(glist); var bnodes = new List <BNode>(); int cntr = 0; foreach (var item in graphList) { cntr++; //note older frameworks will not be in the priority order var main = item.ToString(); if (cntr == 1 || main.IndexOf("skos:ConceptScheme") > -1) { //HACK if (main.IndexOf("skos:ConceptScheme") > -1) { input = JsonConvert.DeserializeObject <InputEntity>(main); } } else { //should just have concepts, but should check for bnodes var child = item.ToString(); if (child.IndexOf("_:") > -1) { bnodes.Add(JsonConvert.DeserializeObject <BNode>(child)); } else if (child.IndexOf("skos:Concept") > -1) { concepts.Add(JsonConvert.DeserializeObject <InputConcept>(child)); } else { //unexpected } } } //try //{ //input = JsonConvert.DeserializeObject<InputGraph>( item.DecodedResource.ToString() ); string ctid = input.CTID; status.Ctid = ctid; string referencedAtId = input.CtdlId; LoggingHelper.DoTrace(5, " ctid: "+ ctid); LoggingHelper.DoTrace(5, " @Id: "+ input.CtdlId); LoggingHelper.DoTrace(5, " name: "+ input.Name.ToString()); string framework = input.Name.ToString(); var org = new MC.Organization(); string orgCTID = ""; string orgName = ""; if (status.DoingDownloadOnly) { return(true); } //add/updating ConceptScheme //var output = new ConceptScheme(); if (!DoesEntityExist(input.CTID, ref output)) { //set the rowid now, so that can be referenced as needed output.RowId = Guid.NewGuid(); //output.RowId = Guid.NewGuid(); } helper.currentBaseObject = output; // output.Name = helper.HandleLanguageMap(input.Name, output, "Name"); output.Description = helper.HandleLanguageMap(input.Description, output, "description"); output.CTID = input.CTID; output.PrimaryOrganizationCTID = orgCTID; // var publisher = input.Publisher; output.PublisherUid = helper.MapOrganizationReferenceGuid("ConceptScheme.Publisher", input.Publisher, ref status); output.Creator = helper.MapOrganizationReferenceGuids("ConceptScheme.Creator", input.Creator, ref status); //20-06-11 - need to get creator, publisher, owner where possible // include an org reference with name, swp, and?? //should check creator first? Or will publisher be more likely to have an account Ctid? if (output.Creator != null && output.Creator.Count() > 0) { //get org or pending stub //look up org name org = OrganizationManager.Exists(output.Creator[0]); output.OwnedBy.Add(output.Creator[0]); } else { if (output.PublisherUid != Guid.Empty) { //get org or pending stub //look up org name org = OrganizationManager.Exists(output.PublisherUid); output.OwnedBy.Add(output.PublisherUid); } } // if (org != null && org.Id > 0) { orgName = org.Name; output.OrganizationId = org.Id; helper.CurrentOwningAgentUid = org.RowId; } output.CredentialRegistryId = envelopeIdentifier; output.HasConcepts = new List <MC.Concept>(); //?store concepts in string? if (concepts != null && concepts.Count > 0) { output.TotalConcepts = concepts.Count(); foreach (var item in concepts) { var c = new MC.Concept() { PrefLabel = helper.HandleLanguageMap(item.PrefLabel, output, "PrefLabel"), Definition = helper.HandleLanguageMap(item.Definition, output, "Definition"), Notes = helper.HandleLanguageMapList(item.Note, output), CTID = item.CTID }; if (c.Notes != null && c.Notes.Any()) { c.Note = c.Notes[0]; } output.HasConcepts.Add(c); } } //20-07-02 just storing the index ready concepts output.ConceptsStore = JsonConvert.SerializeObject(output.HasConcepts, MappingHelperV3.GetJsonSettings()); //adding common import pattern new ConceptSchemeServices().Import(output, ref status); // return(importSuccessfull); }