public bool GetEnvelopePayload(ReadEnvelope item, SaveStatus status, ref string payload)
{
if (item == null || string.IsNullOrWhiteSpace(item.EnvelopeIdentifier))
{
status.AddError("A valid ReadEnvelope must be provided.");
return(false);
}
//
DateTime createDate = new DateTime();
DateTime envelopeUpdateDate = new DateTime();
if (DateTime.TryParse(item.NodeHeaders.CreatedAt.Replace("UTC", "").Trim(), out createDate))
{
status.EnvelopeCreatedDate = createDate;
}
if (DateTime.TryParse(item.NodeHeaders.UpdatedAt.Replace("UTC", "").Trim(), out envelopeUpdateDate))
{
status.EnvelopeUpdatedDate = envelopeUpdateDate;
}
//
payload = item.DecodedResource.ToString();
string envelopeIdentifier = item.EnvelopeIdentifier;
string ctdlType = RegistryServices.GetResourceType(payload);
//string envelopeUrl = RegistryServices.GetEnvelopeUrl( envelopeIdentifier );
return(true);
}
public bool ProcessEnvelope(ReadEnvelope item, string registryEntityType, int entityTypeId, int cntr, bool doingDownloadOnly = false)
{
bool importSuccessfull = false;
if (item == null || item.DecodedResource == null)
{
return(false);
}
int newImportId = 0;
var status = new SaveStatus
{
DoingDownloadOnly = doingDownloadOnly,
ValidationGroup = string.Format("{0} Import", registryEntityType)
};
DateTime started = DateTime.Now;
DateTime envelopeUpdateDate = new DateTime();
DateTime createDate = new DateTime();
if (DateTime.TryParse(item.NodeHeaders.CreatedAt.Replace("UTC", "").Trim(), out createDate))
{
status.SetEnvelopeCreated(createDate);
}
if (DateTime.TryParse(item.NodeHeaders.UpdatedAt.Replace("UTC", "").Trim(), out envelopeUpdateDate))
{
status.SetEnvelopeUpdated(envelopeUpdateDate);
}
LoggingHelper.DoTrace(2, string.Format("{0}. {1} CTID {2}, Updated: {3} ", cntr, registryEntityType, item.EnvelopeCetermsCtid, envelopeUpdateDate.ToString()));
var messages = new List <string>();
string importError = "";
importSuccessfull = false;
//var documentPublishedBy = item.documentPublishedBy ?? ""
//======================================================
//21-01-28 mp - moving common code back here to improve maintenance
if (item.documentPublishedBy != null)
{
//only providing DocumentPublishedBy where apparantly a 3PP
if (item.documentOwnedBy == null || (item.documentPublishedBy != item.documentOwnedBy))
{
status.DocumentPublishedBy = item.documentPublishedBy;
}
}
else
{
//will need to check elsewhere
//OR as part of import check if existing one had 3rd party publisher
}
string payload = item.DecodedResource.ToString();
status.EnvelopeId = item.EnvelopeIdentifier;
var ctid = item.EnvelopeCetermsCtid;
var envelopeCtdlType = item.EnvelopeCtdlType;
string ctdlType = RegistryServices.GetResourceType(payload);
//string envelopeUrl = RegistryServices.GetEnvelopeUrl( envelopeIdentifier );
LoggingHelper.WriteLogFile(UtilityManager.GetAppKeyValue("logFileTraceLevel", 5), item.EnvelopeCetermsCtid + "_" + ctdlType, payload, "", false);
//
try
{
switch (entityTypeId)
{
case 1:
//importSuccessfull = credImportMgr.ProcessEnvelope( item, status );
if (ctdlType.IndexOf("Organization") > -1 || ctdlType.IndexOf("LearningOpportunity") > -1 || ctdlType.IndexOf("Assessment") > -1)
{
//how can this happen????
//- expected as the FTS search will also search blank nodes
importError = string.Format("*****WHILE DOING A CREDENTIAL IMPORT (#{0}), A RECORD OF TYPE: '{1}', CTID: '{2}' WAS ENCOUNTERED! *********************", cntr, ctdlType, ctid);
//status.AddError( importError );
//LoggingHelper.DoTrace( 1, importError );
}
//else if ( ctdlType != envelopeCtdlType )
//{
// LoggingHelper.DoTrace( 1, "___skipping blank node" );
//}
else
{
importSuccessfull = credImportMgr.ImportV3(payload, status);
}
break;
case 2:
//importSuccessfull = orgImportMgr.ProcessEnvelope( item, status );
if (ctdlType.IndexOf("Organization") > -1)
{
importSuccessfull = orgImportMgr.ImportV3(payload, status);
}
else
{
//how can this happen????
importError = string.Format("*****WHILE DOING AN ORGANIZATION IMPORT (#{0}), A RECORD OF TYPE: '{1}', CTID: '{2}' WAS ENCOUNTERED! *********************", cntr, ctdlType, ctid);
status.AddError(importError);
LoggingHelper.DoTrace(1, importError);
}
break;
case 3:
if (ctdlType.IndexOf("Assessment") > -1)
{
importSuccessfull = asmtImportMgr.ProcessEnvelope(item, status);
}
else
{
//how can this happen????
importError = string.Format("*****WHILE DOING AN Assessment IMPORT (#{0}), A RECORD OF TYPE: '{1}', CTID: '{2}' WAS ENCOUNTERED! *********************", cntr, ctdlType, ctid);
status.AddError(importError);
LoggingHelper.DoTrace(1, importError);
}
break;
case 7:
if (ctdlType.IndexOf("LearningOpportunity") > -1)
{
importSuccessfull = loppImportMgr.ProcessEnvelope(item, status);
}
else
{
//how can this happen????
//importError = string.Format( "*****WHILE DOING A LearningOpportunity IMPORT (#{0}), A RECORD OF TYPE: '{1}', CTID: '{2}' WAS ENCOUNTERED! *********************", cntr, ctdlType, ctid );
//status.AddError( importError );
//LoggingHelper.DoTrace( 1, importError );
}
break;
case 8: //
if (ctdlType.IndexOf("Pathway") > -1)
{
importSuccessfull = pathwayImportMgr.ProcessEnvelope(item, status);
}
else
{
//how can this happen????
//importError = string.Format( "*****WHILE DOING A Pathway IMPORT (#{0}), A RECORD OF TYPE: '{1}', CTID: '{2}' WAS ENCOUNTERED! *********************", cntr, ctdlType, ctid );
//status.AddError( importError );
//LoggingHelper.DoTrace( 1, importError );
}
break;
case 9: //
DisplayMessages(string.Format("{0}. Rubrics ({1}) are not handled at this time. ", cntr, entityTypeId));
return(true);
case 10:
case 17:
if (ctdlType.IndexOf("CompetencyFramework") > -1)
{
importSuccessfull = cfImportMgr.ProcessEnvelope(item, status);
}
else
{
//how can this happen????
//importError = string.Format( "*****WHILE DOING A CompetencyFramework IMPORT (#{0}), A RECORD OF TYPE: '{1}', CTID: '{2}' WAS ENCOUNTERED! *********************", cntr, ctdlType, ctid );
//status.AddError( importError );
//LoggingHelper.DoTrace( 1, importError );
}
break;
case 11: //concept scheme
//DisplayMessages( string.Format( "{0}. Concept Schemes ({1}) are not handled at this time. ", cntr, entityTypeId ) );
importSuccessfull = new ImportConceptSchemes().ProcessEnvelope(item, status);
return(true);
case 19:
importSuccessfull = cndManImportMgr.ProcessEnvelope(item, status);
break;
case 20:
importSuccessfull = cstManImportMgr.ProcessEnvelope(item, status);
break;
case 23:
importSuccessfull = new ImportPathwaySets().ProcessEnvelope(item, status);
//DisplayMessages( string.Format( "{0}. PathwaySets ({1}) are not handled at this time. ", cntr, entityTypeId ) );
break;
case 26:
importSuccessfull = tvpImportMgr.ProcessEnvelope(item, status);
//DisplayMessages( string.Format( "{0}. TransferValueProfiles ({1}) are not handled at this time. ", cntr, entityTypeId ) );
break;
default:
DisplayMessages(string.Format("{0}. RegistryImport. Unhandled Entity type encountered: {1} ", cntr, entityTypeId));
break;
}
}
catch (Exception ex)
{
if (ex.Message.IndexOf("Path '@context', line 1") > 0)
{
importError = "The referenced registry document is using an old schema. Please republish it with the latest schema!";
status.AddError(importError);
}
else
{
LoggingHelper.LogError(ex, string.Format(registryEntityType + " Exception encountered in envelopeId: {0}", item.EnvelopeIdentifier), true, "CredentialFinder Import exception");
status.AddError(ex.Message);
importError = ex.Message;
}
//make continue on exceptions an option
//exceptionCtr++;
//if ( maxExceptions > 0 && exceptionCtr > maxExceptions )
//{
// //arbitrarily stop if large number of exceptions
// importNote = string.Format( thisClassName + " - {0} Many exceptions ({1}) were encountered during import - abandoning.", entityType, exceptionCtr );
// //Console.WriteLine( importNote );
// LoggingHelper.DoTrace( 1, importNote );
// LoggingHelper.LogError( importNote, true, thisClassName + "- many exceptions" );
// isComplete = true;
// break;
//}
}
finally
{
if (!importSuccessfull)
{
if (string.IsNullOrWhiteSpace(importError))
{
importError = string.Join("\r\n", status.GetAllMessages().ToArray());
}
}
//store document
//add indicator of success
newImportId = importMgr.Add(item, entityTypeId, status.Ctid, importSuccessfull, importError, ref messages);
if (newImportId > 0 && status.Messages != null && status.Messages.Count > 0)
{
//add indicator of current recored
string msg = string.Format("========= Messages for {4}, EnvelopeIdentifier: {0}, ctid: {1}, Id: {2}, rowId: {3} =========", item.EnvelopeIdentifier, status.Ctid, status.DocumentId, status.DocumentRowId, registryEntityType);
//ensure status has info on the current context, so can be include in messages. Or N/A. The message has the Import.Staging record as the parent
importMgr.AddMessages(newImportId, status, ref messages);
}
TimeSpan duration = DateTime.Now.Subtract(started);
LoggingHelper.DoTrace(2, string.Format(" Total Duration: {0:N2} seconds ", duration.TotalSeconds));
} //finally
return(importSuccessfull);
}
public string Retrieve(string registryEntityType, int entityTypeId, string startingDate, string endingDate, int maxRecords, bool downloadOnly, ref int recordsImported, string sortOrder = "asc")
{
bool importingThisType = UtilityManager.GetAppKeyValue("importing_" + registryEntityType, true);
if (!importingThisType)
{
LoggingHelper.DoTrace(1, string.Format("=== ***************** Skipping import of {0} ***************** ", registryEntityType));
return("Skipped import of " + registryEntityType);
}
LoggingHelper.DoTrace(1, string.Format("=== ***************** Importing {0} ***************** ", registryEntityType));
//JsonEntity input = new JsonEntity();
ReadEnvelope envelope = new ReadEnvelope();
List <ReadEnvelope> list = new List <ReadEnvelope>();
int pageNbr = 1;
int pageSize = UtilityManager.GetAppKeyValue("importPageSize", 100);
string importError = "";
string importResults = "";
string importNote = "";
//ThisEntity output = new ThisEntity();
List <string> messages = new List <string>();
int cntr = 0;
int pTotalRows = 0;
int exceptionCtr = 0;
string statusMessage = "";
bool isComplete = false;
bool importSuccessfull = true;
int newImportId = 0;
//will need to handle multiple calls - watch for time outs
while (pageNbr > 0 && !isComplete)
{
//19-09-22 chg to use RegistryServices to remove duplicate services
list = Search(registryEntityType, startingDate, endingDate, pageNbr, pageSize, ref pTotalRows, ref statusMessage, Community, sortOrder);
if (list == null || list.Count == 0)
{
isComplete = true;
if (pageNbr == 1)
{
LoggingHelper.DoTrace(4, registryEntityType + ": No records where found for date range. ");
}
break;
}
if (pageNbr == 1)
{
LoggingHelper.DoTrace(2, string.Format("Import {0} Found {1} records to process.", registryEntityType, pTotalRows));
}
foreach (ReadEnvelope item in list)
{
cntr++;
string envelopeIdentifier = item.EnvelopeIdentifier;
string ctid = item.EnvelopeCetermsCtid;
string payload = item.DecodedResource.ToString();
LoggingHelper.DoTrace(2, string.Format("{0}. {1} ctid {2} ", cntr, registryEntityType, ctid));
string ctdlType = RegistryServices.GetResourceType(payload);
string envelopeUrl = RegistryHelper.GetEnvelopeUrl(envelopeIdentifier);
//LoggingHelper.DoTrace( 5, " envelopeUrl: " + envelopeUrl );
//to overwrite an existing file, suppress the date prefix (" "), or use an alternate prefix
LoggingHelper.WriteLogFile(1, registryEntityType + "_" + ctid, payload, "", false);
if (maxRecords > 0 && cntr >= maxRecords)
{
break;
}
} //end foreach
pageNbr++;
if ((maxRecords > 0 && cntr >= maxRecords))
{
isComplete = true;
LoggingHelper.DoTrace(2, string.Format("Import {2} EARLY EXIT. Completed {0} records out of a total of {1} for {2} ", cntr, pTotalRows, registryEntityType));
}
else if (cntr >= pTotalRows)
{
isComplete = true;
}
}
importResults = string.Format("Import {0} - Processed {1} records, with {2} exceptions. \r\n", registryEntityType, cntr, exceptionCtr);
LoggingHelper.DoTrace(2, importResults);
if (!string.IsNullOrWhiteSpace(importNote))
{
importResults += importNote;
}
recordsImported += cntr;
return(importResults);
}
/// <summary>
/// Handle deleted records for the requested time period
/// Save to file systems with prefix of Deleted
/// </summary>
/// <param name="community"></param>
/// <param name="maxRecords"></param>
/// <param name="recordsDeleted"></param>
/// <returns></returns>
public string HandleDeletes(string community, int maxRecords, ref int recordsDeleted)
{
int pageNbr = 1;
int pageSize = 50;
//string importError = "";
//may want to just do all types!
string type = "";
List <string> messages = new List <string>();
List <ReadEnvelope> list = new List <ReadEnvelope>();
SaveStatus status = new SaveStatus();
int pTotalRows = 0;
int cntr = 0;
bool isComplete = false;
int exceptionCtr = 0;
string statusMessage = "";
string importResults = "";
string importNote = "";
LoggingHelper.DoTrace(1, string.Format("=== DELETE Check for: '{0}' to '{1}' ===", StartingDate, EndingDate));
//registryImport.StartingDate = "2017-10-29T00:00:00";
try
{
while (pageNbr > 0 && !isComplete)
{
list = GetDeleted(community, type, StartingDate, EndingDate, pageNbr, pageSize, ref pTotalRows, ref statusMessage);
if (list == null || list.Count == 0)
{
isComplete = true;
if (pageNbr == 1)
{
importNote = "Deletes: No records where found for date range ";
//Console.WriteLine( thisClassName + ".HandleDeletes() - " + importNote );
LoggingHelper.DoTrace(1, string.Format("Download.Main.HandleDeletes() Community: {0} - ", community) + importNote);
}
break;
}
foreach (ReadEnvelope item in list)
{
cntr++;
string payload = item.DecodedResource.ToString();
string ctdlType = RegistryServices.GetResourceType(payload);
string ctid = item.EnvelopeCtid;
LoggingHelper.DoTrace(6, string.Format("{0}. ctdlType: {1} ctid: {2} ", cntr, ctdlType, ctid));
try
{
//only need the envelopeId and type
//so want a full delete, or set EntityStateId to 0 - just as a precaution
messages = new List <string>();
//action - place in a deleted folder?
LoggingHelper.WriteLogFile(1, "Deleted_" + ctdlType + "_" + ctid, payload, "", false);
}
catch (Exception ex)
{
LoggingHelper.LogError(ex, string.Format("Exception encountered in envelopeId: {0}", item.EnvelopeIdentifier), false, "CredentialFinder Import exception");
//importError = ex.Message;
}
if (maxRecords > 0 && cntr > maxRecords)
{
break;
}
} //foreach ( ReadEnvelope item in list )
pageNbr++;
if ((maxRecords > 0 && cntr > maxRecords) || cntr > pTotalRows)
{
isComplete = true;
DisplayMessages(string.Format("Delete EARLY EXIT. Completed {0} records out of a total of {1} ", cntr, pTotalRows));
}
} //while
//delete from elastic
if (cntr > 0)
{
messages = new List <string>();
}
importResults = string.Format("HandleDeletes - Processed {0} records, with {1} exceptions. \r\n", cntr, exceptionCtr);
if (!string.IsNullOrWhiteSpace(importNote))
{
importResults += importNote;
}
}
catch (Exception ex)
{
LoggingHelper.LogError(ex, "Import.HandleDeletes");
}
//actually only attepted at this time, need to account for errors!
recordsDeleted = cntr;
return(importResults);
}
public static string HandleDeletes(string community, string startingDate, string endingDate, int maxRecords, ref int recordsDeleted)
{
int pageNbr = 1;
int pageSize = 50;
//string importError = "";
//may want to just do all types!
string type = "";
List <string> messages = new List <string>();
List <ReadEnvelope> list = new List <ReadEnvelope>();
SaveStatus status = new SaveStatus();
int pTotalRows = 0;
int cntr = 0;
bool isComplete = false;
int exceptionCtr = 0;
string statusMessage = "";
string importResults = "";
string importNote = "";
LoggingHelper.DoTrace(1, string.Format("=== DELETES ===", thisClassName));
//startingDate = "2017-10-29T00:00:00";
try
{
while (pageNbr > 0 && !isComplete)
{
list = RegistryImport.GetDeleted(community, type, startingDate, endingDate, pageNbr, pageSize, ref pTotalRows, ref statusMessage);
if (list == null || list.Count == 0)
{
isComplete = true;
if (pageNbr == 1)
{
importNote = "Deletes: No records where found for date range ";
//Console.WriteLine( thisClassName + ".HandleDeletes() - " + importNote );
LoggingHelper.DoTrace(1, thisClassName + ".HandleDeletes() - " + importNote);
}
break;
}
foreach (ReadEnvelope item in list)
{
cntr++;
string payload = item.DecodedResource.ToString();
string ctdlType = RegistryServices.GetResourceType(payload);
string ctid = RegistryServices.GetCtidFromUnknownEnvelope(item);
//may not be available in database, may want to use ctid
string envelopeIdentifier = item.EnvelopeIdentifier;
string envelopeUrl = RegistryServices.GetEnvelopeUrl(item.EnvelopeIdentifier);
LoggingHelper.DoTrace(5, " envelopeUrl: "+ envelopeUrl);
LoggingHelper.DoTrace(6, string.Format("{0}. EnvelopeIdentifier: {1} ", cntr, item.EnvelopeIdentifier));
try
{
//only need the envelopeId and type
//so want a full delete, or set EntityStateId to 4 or greater - just as a precaution
messages = new List <string>();
status = new SaveStatus();
status.ValidationGroup = "Deletes";
//importError = "";
//each delete method will add an entry to SearchPendingReindex.
//at the end of the process, call method to handle all the deletes
switch (ctdlType.ToLower())
{
case "credentialorganization":
case "qacredentialorganization":
case "organization":
DisplayMessages(string.Format("{0}. Deleting {3} by EnvelopeIdentifier/ctid: {1}/{2} ", cntr, item.EnvelopeIdentifier, ctid, ctdlType));
if (!new OrganizationManager().Delete(envelopeIdentifier, ctid, ref statusMessage))
{
DisplayMessages(string.Format(" Delete failed: {0} ", statusMessage));
}
break;
case "assessmentprofile":
DisplayMessages(string.Format("{0}. Deleting Assessment by EnvelopeIdentifier/ctid: {1}/{2} ", cntr, item.EnvelopeIdentifier, ctid));
if (!new AssessmentManager().Delete(envelopeIdentifier, ctid, ref statusMessage))
{
DisplayMessages(string.Format(" Delete failed: {0} ", statusMessage));
}
break;
case "learningopportunityprofile":
DisplayMessages(string.Format("{0}. Deleting LearningOpportunity by EnvelopeIdentifier/ctid: {1}/{2} ", cntr, item.EnvelopeIdentifier, ctid));
if (!new LearningOpportunityManager().Delete(envelopeIdentifier, ctid, ref statusMessage))
{
DisplayMessages(string.Format(" Delete failed: {0} ", statusMessage));
}
break;
case "conditionmanifest":
DisplayMessages(string.Format("{0}. Deleting ConditionManifest by EnvelopeIdentifier/ctid: {1}/{2} ", cntr, item.EnvelopeIdentifier, ctid));
if (!new ConditionManifestManager().Delete(envelopeIdentifier, ctid, ref statusMessage))
{
DisplayMessages(string.Format(" Delete failed: {0} ", statusMessage));
}
break;
case "costmanifest":
DisplayMessages(string.Format("{0}. Deleting CostManifest by EnvelopeIdentifier/ctid: {1}/{2} ", cntr, item.EnvelopeIdentifier, ctid));
if (!new CostManifestManager().Delete(envelopeIdentifier, ctid, ref statusMessage))
{
DisplayMessages(string.Format(" Delete failed: {0} ", statusMessage));
}
break;
case "competencyframework": //CompetencyFramework
DisplayMessages(string.Format("{0}. Deleting CompetencyFramework by EnvelopeIdentifier/ctid: {1}/{2} ", cntr, item.EnvelopeIdentifier, ctid));
if (!new EducationFrameworkManager().Delete(envelopeIdentifier, ctid, ref statusMessage))
{
DisplayMessages(string.Format(" Delete failed: {0} ", statusMessage));
}
break;
default:
//default to credential
DisplayMessages(string.Format("{0}. Deleting Credential ({2}) by EnvelopeIdentifier/ctid: {1}/{3} ", cntr, item.EnvelopeIdentifier, ctdlType, ctid));
if (!new CredentialManager().Delete(envelopeIdentifier, ctid, ref statusMessage))
{
DisplayMessages(string.Format(" Delete failed: {0} ", statusMessage));
}
break;
}
}
catch (Exception ex)
{
LoggingHelper.LogError(ex, string.Format("Exception encountered in envelopeId: {0}", item.EnvelopeIdentifier), false, "CredentialFinder Import exception");
//importError = ex.Message;
}
if (maxRecords > 0 && cntr > maxRecords)
{
break;
}
} //foreach ( ReadEnvelope item in list )
pageNbr++;
if ((maxRecords > 0 && cntr > maxRecords) || cntr > pTotalRows)
{
isComplete = true;
DisplayMessages(string.Format("Delete EARLY EXIT. Completed {0} records out of a total of {1} ", cntr, pTotalRows));
}
} //while
//delete from elastic
if (cntr > 0)
{
messages = new List <string>();
ElasticServices.HandlePendingDeletes(ref messages);
}
importResults = string.Format("HandleDeletes - Processed {0} records, with {1} exceptions. \r\n", cntr, exceptionCtr);
if (!string.IsNullOrWhiteSpace(importNote))
{
importResults += importNote;
}
}
catch (Exception ex)
{
LoggingHelper.LogError(ex, "Import.HandleDeletes");
}
//actually only attepted at this time, need to account for errors!
recordsDeleted = cntr;
return(importResults);
}