public string Import(string registryEntityType, int entityTypeId, string startingDate, string endingDate, int maxRecords, bool downloadOnly, ref int recordsImported) { bool importingThisType = UtilityManager.GetAppKeyValue("importing_" + registryEntityType, true); if (!importingThisType) { LoggingHelper.DoTrace(1, string.Format("=== ***************** Skipping import of {0} ***************** ", registryEntityType)); return("Skipped import of " + registryEntityType); } LoggingHelper.DoTrace(1, string.Format("=== ***************** Importing {0} ***************** ", registryEntityType)); //JsonEntity input = new JsonEntity(); ReadEnvelope envelope = new ReadEnvelope(); List <ReadEnvelope> list = new List <ReadEnvelope>(); //EntityServices mgr = new EntityServices(); ImportServiceHelpers importMgr = new ImportServiceHelpers(); string entityType = registryEntityType; CodeItem ci = CodesManager.Codes_EntityType_Get(entityTypeId); if (ci != null && ci.Id > 0) { entityType = ci.Title; } int pageNbr = 1; int pageSize = 25; string importError = ""; string importResults = ""; string importNote = ""; //ThisEntity output = new ThisEntity(); List <string> messages = new List <string>(); int cntr = 0; int pTotalRows = 0; int exceptionCtr = 0; string statusMessage = ""; bool isComplete = false; bool importSuccessfull = true; int newImportId = 0; SaveStatus status = new SaveStatus(); //will need to handle multiple calls - watch for time outs while (pageNbr > 0 && !isComplete) { //19-09-22 chg to use RegistryServices to remove duplicate services list = RegistryServices.Search(registryEntityType, startingDate, endingDate, pageNbr, pageSize, ref pTotalRows, ref statusMessage, Community); //list = RegistryImport.GetLatest( registryEntityType, startingDate, endingDate, pageNbr, pageSize, ref pTotalRows, ref statusMessage, Community ); if (list == null || list.Count == 0) { isComplete = true; if (pageNbr == 1) { //importNote = registryEntityType + ": No records where found for date range "; //Console.WriteLine( thisClassName + importNote ); LoggingHelper.DoTrace(4, registryEntityType + ": No records where found for date range. "); } break; } foreach (ReadEnvelope item in list) { cntr++; string payload = item.DecodedResource.ToString(); newImportId = 0; LoggingHelper.DoTrace(2, string.Format("{0}. {1} EnvelopeIdentifier {2} ", cntr, registryEntityType, item.EnvelopeIdentifier)); messages = new List <string>(); status = new SaveStatus(); status.DoingDownloadOnly = downloadOnly; status.ValidationGroup = string.Format("{0} Import", registryEntityType); importError = ""; importSuccessfull = false; try { //Console.WriteLine( string.Format( "{0}. {1} EnvelopeIdentifier {2} ", cntr, registryEntityType, item.EnvelopeIdentifier ) ); if (payload.IndexOf("@graphXXX") > 0) { DisplayMessages(string.Format("{0}. {1} Import Encountered an envelope using @graph which is not handled yet: {2} ", cntr, entityTypeId, item.EnvelopeIdentifier)); } else { switch (entityTypeId) { case 1: importSuccessfull = credImportMgr.ProcessEnvelope(item, status); break; case 2: importSuccessfull = orgImportMgr.ProcessEnvelope(item, status); break; case 3: importSuccessfull = asmtImportMgr.ProcessEnvelope(item, status); break; case 7: importSuccessfull = loppImportMgr.ProcessEnvelope(item, status); break; case 19: importSuccessfull = cndManImportMgr.ProcessEnvelope(item, status); break; case 20: importSuccessfull = cstManImportMgr.ProcessEnvelope(item, status); break; default: DisplayMessages(string.Format("{0}. Invalid Entity type encountered: {1} ", cntr, entityTypeId)); break; } } } catch (Exception ex) { if (ex.Message.IndexOf("Path '@context', line 1") > 0) { importError = "The referenced registry document is using an old schema. Please republish it with the latest schema!"; status.AddError(importError); } else { LoggingHelper.LogError(ex, string.Format(entityType + " Exception encountered in envelopeId: {0}", item.EnvelopeIdentifier), false, "CredentialFinder Import exception"); status.AddError(ex.Message); importError = ex.Message; } //make continue on exceptions an option exceptionCtr++; if (maxExceptions > 0 && exceptionCtr > maxExceptions) { //arbitrarily stop if large number of exceptions importNote = string.Format(thisClassName + " - {0} Many exceptions ({1}) were encountered during import - abandoning.", entityType, exceptionCtr); //Console.WriteLine( importNote ); LoggingHelper.DoTrace(1, importNote); LoggingHelper.LogError(importNote, true, thisClassName + "- many exceptions"); isComplete = true; break; } } finally { if (!importSuccessfull) { if (string.IsNullOrWhiteSpace(importError)) { importError = string.Join("\r\n", status.GetAllMessages().ToArray()); } } //store document //add indicator of success newImportId = importMgr.Add(item, entityTypeId, status.Ctid, importSuccessfull, importError, ref messages); if (newImportId > 0 && status.Messages != null && status.Messages.Count > 0) { //add indicator of current recored string msg = string.Format("========= Messages for {4}, EnvelopeIdentifier: {0}, ctid: {1}, Id: {2}, rowId: {3} =========", item.EnvelopeIdentifier, status.Ctid, status.DocumentId, status.DocumentRowId, registryEntityType); importMgr.AddMessages(newImportId, status, ref messages); } } //finally if (maxRecords > 0 && cntr > maxRecords) { break; } } //end foreach pageNbr++; if ((maxRecords > 0 && cntr > maxRecords) || cntr > pTotalRows) { isComplete = true; LoggingHelper.DoTrace(2, string.Format("Import {2} EARLY EXIT. Completed {0} records out of a total of {1} ", cntr, pTotalRows, registryEntityType)); } //if ( pageNbr * pageSize < pTotalRows ) // pageNbr++; //else // isComplete = true; } importResults = string.Format("Import {0} - Processed {1} records, with {2} exceptions. \r\n", registryEntityType, cntr, exceptionCtr); if (!string.IsNullOrWhiteSpace(importNote)) { importResults += importNote; } recordsImported += cntr; return(importResults); }
public bool ProcessEnvelope(ReadEnvelope item, string registryEntityType, int entityTypeId, int cntr, bool doingDownloadOnly = false) { bool importSuccessfull = false; if (item == null || item.DecodedResource == null) { return(false); } int newImportId = 0; var status = new SaveStatus { DoingDownloadOnly = doingDownloadOnly, ValidationGroup = string.Format("{0} Import", registryEntityType) }; DateTime started = DateTime.Now; DateTime envelopeUpdateDate = new DateTime(); DateTime createDate = new DateTime(); if (DateTime.TryParse(item.NodeHeaders.CreatedAt.Replace("UTC", "").Trim(), out createDate)) { status.SetEnvelopeCreated(createDate); } if (DateTime.TryParse(item.NodeHeaders.UpdatedAt.Replace("UTC", "").Trim(), out envelopeUpdateDate)) { status.SetEnvelopeUpdated(envelopeUpdateDate); } LoggingHelper.DoTrace(2, string.Format("{0}. {1} CTID {2}, Updated: {3} ", cntr, registryEntityType, item.EnvelopeCetermsCtid, envelopeUpdateDate.ToString())); var messages = new List <string>(); string importError = ""; importSuccessfull = false; //var documentPublishedBy = item.documentPublishedBy ?? "" //====================================================== //21-01-28 mp - moving common code back here to improve maintenance if (item.documentPublishedBy != null) { //only providing DocumentPublishedBy where apparantly a 3PP if (item.documentOwnedBy == null || (item.documentPublishedBy != item.documentOwnedBy)) { status.DocumentPublishedBy = item.documentPublishedBy; } } else { //will need to check elsewhere //OR as part of import check if existing one had 3rd party publisher } string payload = item.DecodedResource.ToString(); status.EnvelopeId = item.EnvelopeIdentifier; var ctid = item.EnvelopeCetermsCtid; var envelopeCtdlType = item.EnvelopeCtdlType; string ctdlType = RegistryServices.GetResourceType(payload); //string envelopeUrl = RegistryServices.GetEnvelopeUrl( envelopeIdentifier ); LoggingHelper.WriteLogFile(UtilityManager.GetAppKeyValue("logFileTraceLevel", 5), item.EnvelopeCetermsCtid + "_" + ctdlType, payload, "", false); // try { switch (entityTypeId) { case 1: //importSuccessfull = credImportMgr.ProcessEnvelope( item, status ); if (ctdlType.IndexOf("Organization") > -1 || ctdlType.IndexOf("LearningOpportunity") > -1 || ctdlType.IndexOf("Assessment") > -1) { //how can this happen???? //- expected as the FTS search will also search blank nodes importError = string.Format("*****WHILE DOING A CREDENTIAL IMPORT (#{0}), A RECORD OF TYPE: '{1}', CTID: '{2}' WAS ENCOUNTERED! *********************", cntr, ctdlType, ctid); //status.AddError( importError ); //LoggingHelper.DoTrace( 1, importError ); } //else if ( ctdlType != envelopeCtdlType ) //{ // LoggingHelper.DoTrace( 1, "___skipping blank node" ); //} else { importSuccessfull = credImportMgr.ImportV3(payload, status); } break; case 2: //importSuccessfull = orgImportMgr.ProcessEnvelope( item, status ); if (ctdlType.IndexOf("Organization") > -1) { importSuccessfull = orgImportMgr.ImportV3(payload, status); } else { //how can this happen???? importError = string.Format("*****WHILE DOING AN ORGANIZATION IMPORT (#{0}), A RECORD OF TYPE: '{1}', CTID: '{2}' WAS ENCOUNTERED! *********************", cntr, ctdlType, ctid); status.AddError(importError); LoggingHelper.DoTrace(1, importError); } break; case 3: if (ctdlType.IndexOf("Assessment") > -1) { importSuccessfull = asmtImportMgr.ProcessEnvelope(item, status); } else { //how can this happen???? importError = string.Format("*****WHILE DOING AN Assessment IMPORT (#{0}), A RECORD OF TYPE: '{1}', CTID: '{2}' WAS ENCOUNTERED! *********************", cntr, ctdlType, ctid); status.AddError(importError); LoggingHelper.DoTrace(1, importError); } break; case 7: if (ctdlType.IndexOf("LearningOpportunity") > -1) { importSuccessfull = loppImportMgr.ProcessEnvelope(item, status); } else { //how can this happen???? //importError = string.Format( "*****WHILE DOING A LearningOpportunity IMPORT (#{0}), A RECORD OF TYPE: '{1}', CTID: '{2}' WAS ENCOUNTERED! *********************", cntr, ctdlType, ctid ); //status.AddError( importError ); //LoggingHelper.DoTrace( 1, importError ); } break; case 8: // if (ctdlType.IndexOf("Pathway") > -1) { importSuccessfull = pathwayImportMgr.ProcessEnvelope(item, status); } else { //how can this happen???? //importError = string.Format( "*****WHILE DOING A Pathway IMPORT (#{0}), A RECORD OF TYPE: '{1}', CTID: '{2}' WAS ENCOUNTERED! *********************", cntr, ctdlType, ctid ); //status.AddError( importError ); //LoggingHelper.DoTrace( 1, importError ); } break; case 9: // DisplayMessages(string.Format("{0}. Rubrics ({1}) are not handled at this time. ", cntr, entityTypeId)); return(true); case 10: case 17: if (ctdlType.IndexOf("CompetencyFramework") > -1) { importSuccessfull = cfImportMgr.ProcessEnvelope(item, status); } else { //how can this happen???? //importError = string.Format( "*****WHILE DOING A CompetencyFramework IMPORT (#{0}), A RECORD OF TYPE: '{1}', CTID: '{2}' WAS ENCOUNTERED! *********************", cntr, ctdlType, ctid ); //status.AddError( importError ); //LoggingHelper.DoTrace( 1, importError ); } break; case 11: //concept scheme //DisplayMessages( string.Format( "{0}. Concept Schemes ({1}) are not handled at this time. ", cntr, entityTypeId ) ); importSuccessfull = new ImportConceptSchemes().ProcessEnvelope(item, status); return(true); case 19: importSuccessfull = cndManImportMgr.ProcessEnvelope(item, status); break; case 20: importSuccessfull = cstManImportMgr.ProcessEnvelope(item, status); break; case 23: importSuccessfull = new ImportPathwaySets().ProcessEnvelope(item, status); //DisplayMessages( string.Format( "{0}. PathwaySets ({1}) are not handled at this time. ", cntr, entityTypeId ) ); break; case 26: importSuccessfull = tvpImportMgr.ProcessEnvelope(item, status); //DisplayMessages( string.Format( "{0}. TransferValueProfiles ({1}) are not handled at this time. ", cntr, entityTypeId ) ); break; default: DisplayMessages(string.Format("{0}. RegistryImport. Unhandled Entity type encountered: {1} ", cntr, entityTypeId)); break; } } catch (Exception ex) { if (ex.Message.IndexOf("Path '@context', line 1") > 0) { importError = "The referenced registry document is using an old schema. Please republish it with the latest schema!"; status.AddError(importError); } else { LoggingHelper.LogError(ex, string.Format(registryEntityType + " Exception encountered in envelopeId: {0}", item.EnvelopeIdentifier), true, "CredentialFinder Import exception"); status.AddError(ex.Message); importError = ex.Message; } //make continue on exceptions an option //exceptionCtr++; //if ( maxExceptions > 0 && exceptionCtr > maxExceptions ) //{ // //arbitrarily stop if large number of exceptions // importNote = string.Format( thisClassName + " - {0} Many exceptions ({1}) were encountered during import - abandoning.", entityType, exceptionCtr ); // //Console.WriteLine( importNote ); // LoggingHelper.DoTrace( 1, importNote ); // LoggingHelper.LogError( importNote, true, thisClassName + "- many exceptions" ); // isComplete = true; // break; //} } finally { if (!importSuccessfull) { if (string.IsNullOrWhiteSpace(importError)) { importError = string.Join("\r\n", status.GetAllMessages().ToArray()); } } //store document //add indicator of success newImportId = importMgr.Add(item, entityTypeId, status.Ctid, importSuccessfull, importError, ref messages); if (newImportId > 0 && status.Messages != null && status.Messages.Count > 0) { //add indicator of current recored string msg = string.Format("========= Messages for {4}, EnvelopeIdentifier: {0}, ctid: {1}, Id: {2}, rowId: {3} =========", item.EnvelopeIdentifier, status.Ctid, status.DocumentId, status.DocumentRowId, registryEntityType); //ensure status has info on the current context, so can be include in messages. Or N/A. The message has the Import.Staging record as the parent importMgr.AddMessages(newImportId, status, ref messages); } TimeSpan duration = DateTime.Now.Subtract(started); LoggingHelper.DoTrace(2, string.Format(" Total Duration: {0:N2} seconds ", duration.TotalSeconds)); } //finally return(importSuccessfull); }
public string Import(string startingDate, string endingDate, int maxRecords, string community, bool downloadOnly = false) { bool importingThisType = UtilityManager.GetAppKeyValue("importing_competency_frameworks", true); if (!importingThisType) { LoggingHelper.DoTrace(1, "=== ***************** Skipping import of Competency Frameworks ***************** "); return("Skipped import of competency_frameworks"); } LoggingHelper.DoTrace(1, string.Format("=== ***************** {0} ***************** ", thisClassName)); JsonEntity input = new JsonEntity(); ReadEnvelope envelope = new ReadEnvelope(); List <ReadEnvelope> list = new List <ReadEnvelope>(); EntityServices mgr = new EntityServices(); string type = "competency_framework"; int pageNbr = 1; int pageSize = 75; string importError = ""; string importResults = ""; string importNote = ""; ThisEntity output = new ThisEntity(); List <string> messages = new List <string>(); //the nbr of records needs to be monitored, to determine the optimum //NOTE: consider the IOER approach that all candidate records are first downloaded, and then a separate process does the import int cntr = 0; int pTotalRows = 0; int exceptionCtr = 0; string statusMessage = ""; bool isComplete = false; bool importSuccessfull = true; int newImportId = 0; SaveStatus status = new SaveStatus(); //will need to handle multiple calls - watch for time outs while (pageNbr > 0 && !isComplete) { list = RegistryServices.Search(type, startingDate, endingDate, pageNbr, pageSize, ref pTotalRows, ref statusMessage, community); //list = GetLatest( type, startingDate, endingDate, pageNbr, pageSize, ref pTotalRows, ref statusMessage, community ); if (list == null || list.Count == 0) { isComplete = true; if (pageNbr == 1) { importNote = "Competency Frameworks: No records where found for date range "; //Console.WriteLine( thisClassName + importNote ); LoggingHelper.DoTrace(4, thisClassName + importNote); } break; } foreach (ReadEnvelope item in list) { cntr++; string payload = item.DecodedResource.ToString(); LoggingHelper.DoTrace(2, string.Format("{0}. EnvelopeIdentifier: {1} ", cntr, item.EnvelopeIdentifier)); status = new SaveStatus(); status.DoingDownloadOnly = downloadOnly; importError = ""; importSuccessfull = false; try { //Console.WriteLine( string.Format( "{0}. Competency Frameswork Envelope Identifier {1} ", cntr, item.EnvelopeIdentifier ) ); importSuccessfull = entityImportMgr.ProcessEnvelope(mgr, item, status); } catch (Exception ex) { if (ex.Message.IndexOf("Path '@context', line 1") > 0) { importError = "The referenced registry document is using an old schema. Please republish it with the latest schema!"; status.AddError(importError); } else { LoggingHelper.LogError(ex, string.Format("Exception encountered in envelopeId: {0}", item.EnvelopeIdentifier), false, "CredentialFinder Import exception"); status.AddError(ex.Message); importError = ex.Message; LoggingHelper.DoTrace(1, " show exception to use getting related envelopeId " + ex.Message); } //make continue on exceptions an option exceptionCtr++; if (maxExceptions > 0 && exceptionCtr > maxExceptions) { //arbitrarily stop if large number of exceptions importNote = string.Format(thisClassName + " - Many exceptions ({0}) were encountered during import - abandoning.", exceptionCtr); Console.WriteLine(importNote); LoggingHelper.DoTrace(1, importNote); LoggingHelper.LogError(importNote, true, thisClassName + "- many exceptions"); isComplete = true; break; } } finally { if (!importSuccessfull) { if (string.IsNullOrWhiteSpace(importError)) { importError = string.Join("\r\n", status.GetAllMessages().ToArray()); } } //store document //add indicator of success newImportId = importMgr.Add(item, entityTypeId, status.Ctid, importSuccessfull, importError, ref messages); if (newImportId > 0 && status.Messages != null && status.Messages.Count > 0) { importMgr.AddMessages(newImportId, status, ref messages); } } //finally if (maxRecords > 0 && cntr > maxRecords) { break; } } //end foreach pageNbr++; if ((maxRecords > 0 && cntr > maxRecords) || cntr == pTotalRows) { isComplete = true; if (cntr < pTotalRows) { LoggingHelper.DoTrace(2, string.Format("CompetencyFramesworkImport EARLY EXIT. Completed {0} records out of a total of {1} ", cntr, pTotalRows)); } } } // importResults = string.Format("{0} - Processed {1} records, with {2} exceptions. \r\n", thisClassName, cntr, exceptionCtr); if (!string.IsNullOrWhiteSpace(importNote)) { importResults += importNote; } //always call, as deletes are not tracked if (UtilityManager.GetAppKeyValue("updateCompetencyFrameworkReportTotals", false) == true) { mgr.UpdateCompetencyFrameworkReportTotals(); } return(importResults); }