public string Import(string registryEntityType, int entityTypeId, string startingDate, string endingDate, int maxRecords, bool downloadOnly, ref int recordsImported)
{
bool importingThisType = UtilityManager.GetAppKeyValue("importing_" + registryEntityType, true);
if (!importingThisType)
{
LoggingHelper.DoTrace(1, string.Format("=== ***************** Skipping import of {0} ***************** ", registryEntityType));
return("Skipped import of " + registryEntityType);
}
LoggingHelper.DoTrace(1, string.Format("=== ***************** Importing {0} ***************** ", registryEntityType));
//JsonEntity input = new JsonEntity();
ReadEnvelope envelope = new ReadEnvelope();
List <ReadEnvelope> list = new List <ReadEnvelope>();
//EntityServices mgr = new EntityServices();
ImportServiceHelpers importMgr = new ImportServiceHelpers();
string entityType = registryEntityType;
CodeItem ci = CodesManager.Codes_EntityType_Get(entityTypeId);
if (ci != null && ci.Id > 0)
{
entityType = ci.Title;
}
int pageNbr = 1;
int pageSize = 25;
string importError = "";
string importResults = "";
string importNote = "";
//ThisEntity output = new ThisEntity();
List <string> messages = new List <string>();
int cntr = 0;
int pTotalRows = 0;
int exceptionCtr = 0;
string statusMessage = "";
bool isComplete = false;
bool importSuccessfull = true;
int newImportId = 0;
SaveStatus status = new SaveStatus();
//will need to handle multiple calls - watch for time outs
while (pageNbr > 0 && !isComplete)
{
//19-09-22 chg to use RegistryServices to remove duplicate services
list = RegistryServices.Search(registryEntityType, startingDate, endingDate, pageNbr, pageSize, ref pTotalRows, ref statusMessage, Community);
//list = RegistryImport.GetLatest( registryEntityType, startingDate, endingDate, pageNbr, pageSize, ref pTotalRows, ref statusMessage, Community );
if (list == null || list.Count == 0)
{
isComplete = true;
if (pageNbr == 1)
{
//importNote = registryEntityType + ": No records where found for date range ";
//Console.WriteLine( thisClassName + importNote );
LoggingHelper.DoTrace(4, registryEntityType + ": No records where found for date range. ");
}
break;
}
foreach (ReadEnvelope item in list)
{
cntr++;
string payload = item.DecodedResource.ToString();
newImportId = 0;
LoggingHelper.DoTrace(2, string.Format("{0}. {1} EnvelopeIdentifier {2} ", cntr, registryEntityType, item.EnvelopeIdentifier));
messages = new List <string>();
status = new SaveStatus();
status.DoingDownloadOnly = downloadOnly;
status.ValidationGroup = string.Format("{0} Import", registryEntityType);
importError = "";
importSuccessfull = false;
try
{
//Console.WriteLine( string.Format( "{0}. {1} EnvelopeIdentifier {2} ", cntr, registryEntityType, item.EnvelopeIdentifier ) );
if (payload.IndexOf("@graphXXX") > 0)
{
DisplayMessages(string.Format("{0}. {1} Import Encountered an envelope using @graph which is not handled yet: {2} ", cntr, entityTypeId, item.EnvelopeIdentifier));
}
else
{
switch (entityTypeId)
{
case 1:
importSuccessfull = credImportMgr.ProcessEnvelope(item, status);
break;
case 2:
importSuccessfull = orgImportMgr.ProcessEnvelope(item, status);
break;
case 3:
importSuccessfull = asmtImportMgr.ProcessEnvelope(item, status);
break;
case 7:
importSuccessfull = loppImportMgr.ProcessEnvelope(item, status);
break;
case 19:
importSuccessfull = cndManImportMgr.ProcessEnvelope(item, status);
break;
case 20:
importSuccessfull = cstManImportMgr.ProcessEnvelope(item, status);
break;
default:
DisplayMessages(string.Format("{0}. Invalid Entity type encountered: {1} ", cntr, entityTypeId));
break;
}
}
}
catch (Exception ex)
{
if (ex.Message.IndexOf("Path '@context', line 1") > 0)
{
importError = "The referenced registry document is using an old schema. Please republish it with the latest schema!";
status.AddError(importError);
}
else
{
LoggingHelper.LogError(ex, string.Format(entityType + " Exception encountered in envelopeId: {0}", item.EnvelopeIdentifier), false, "CredentialFinder Import exception");
status.AddError(ex.Message);
importError = ex.Message;
}
//make continue on exceptions an option
exceptionCtr++;
if (maxExceptions > 0 && exceptionCtr > maxExceptions)
{
//arbitrarily stop if large number of exceptions
importNote = string.Format(thisClassName + " - {0} Many exceptions ({1}) were encountered during import - abandoning.", entityType, exceptionCtr);
//Console.WriteLine( importNote );
LoggingHelper.DoTrace(1, importNote);
LoggingHelper.LogError(importNote, true, thisClassName + "- many exceptions");
isComplete = true;
break;
}
}
finally
{
if (!importSuccessfull)
{
if (string.IsNullOrWhiteSpace(importError))
{
importError = string.Join("\r\n", status.GetAllMessages().ToArray());
}
}
//store document
//add indicator of success
newImportId = importMgr.Add(item, entityTypeId, status.Ctid, importSuccessfull, importError, ref messages);
if (newImportId > 0 && status.Messages != null && status.Messages.Count > 0)
{
//add indicator of current recored
string msg = string.Format("========= Messages for {4}, EnvelopeIdentifier: {0}, ctid: {1}, Id: {2}, rowId: {3} =========", item.EnvelopeIdentifier, status.Ctid, status.DocumentId, status.DocumentRowId, registryEntityType);
importMgr.AddMessages(newImportId, status, ref messages);
}
} //finally
if (maxRecords > 0 && cntr > maxRecords)
{
break;
}
} //end foreach
pageNbr++;
if ((maxRecords > 0 && cntr > maxRecords) || cntr > pTotalRows)
{
isComplete = true;
LoggingHelper.DoTrace(2, string.Format("Import {2} EARLY EXIT. Completed {0} records out of a total of {1} ", cntr, pTotalRows, registryEntityType));
}
//if ( pageNbr * pageSize < pTotalRows )
// pageNbr++;
//else
// isComplete = true;
}
importResults = string.Format("Import {0} - Processed {1} records, with {2} exceptions. \r\n", registryEntityType, cntr, exceptionCtr);
if (!string.IsNullOrWhiteSpace(importNote))
{
importResults += importNote;
}
recordsImported += cntr;
return(importResults);
}
public string Import(string registryEntityType, int entityTypeId, string startingDate, string endingDate, int maxRecords, bool downloadOnly, ref int recordsImported, string sortOrder = "asc")
{
bool importingThisType = UtilityManager.GetAppKeyValue("importing_" + registryEntityType, true);
if (!importingThisType)
{
LoggingHelper.DoTrace(1, string.Format("=== ***************** Skipping import of {0} ***************** ", registryEntityType));
return("Skipped import of " + registryEntityType);
}
LoggingHelper.DoTrace(1, string.Format("=== ***************** Importing {0} ***************** ", registryEntityType));
//JsonEntity input = new JsonEntity();
ReadEnvelope envelope = new ReadEnvelope();
List <ReadEnvelope> list = new List <ReadEnvelope>();
string entityType = registryEntityType;
CodeItem ci = CodesManager.Codes_EntityType_Get(entityTypeId);
if (ci != null && ci.Id > 0)
{
entityType = ci.Title;
}
int pageNbr = 1;
int pageSize = UtilityManager.GetAppKeyValue("importPageSize", 100);
string importResults = "";
string importNote = "";
//ThisEntity output = new ThisEntity();
List <string> messages = new List <string>();
int cntr = 0;
int actualTotalRows = 0;
int pTotalRows = 0;
int exceptionCtr = 0;
string statusMessage = "";
bool isComplete = false;
bool importSuccessfull = true;
//will need to handle multiple calls - watch for time outs
while (pageNbr > 0 && !isComplete)
{
//19-09-22 chg to use RegistryServices to remove duplicate services
list = RegistryServices.Search(registryEntityType, startingDate, endingDate, pageNbr, pageSize, ref pTotalRows, ref statusMessage, Community, sortOrder);
//list = RegistryImport.GetLatest( registryEntityType, startingDate, endingDate, pageNbr, pageSize, ref pTotalRows, ref statusMessage, Community );
if (list == null || list.Count == 0)
{
isComplete = true;
if (pageNbr == 1)
{
//importNote = registryEntityType + ": No records where found for date range ";
//Console.WriteLine( thisClassName + importNote );
LoggingHelper.DoTrace(4, registryEntityType + ": No records where found for date range. ");
}
else if (cntr < actualTotalRows)
{
//if no data found and have not processed actual rows, could have been an issue with the search.
//perhaps should be an error to ensure followup
LoggingHelper.DoTrace(2, string.Format("**************** WARNING -Import for '{0}' didn't find data on this pass, but has only processed {1} of an expected {2} records.", registryEntityType, cntr, actualTotalRows));
LoggingHelper.LogError(string.Format("**************** WARNING -Import for '{0}' didn't find data on this pass, but has only processed {1} of an expected {2} records.", registryEntityType, cntr, actualTotalRows), true, "Finder Import Ended Early");
}
break;
}
if (pageNbr == 1)
{
actualTotalRows = pTotalRows;
LoggingHelper.DoTrace(2, string.Format("Import {0} Found {1} records to process.", registryEntityType, pTotalRows));
}
foreach (ReadEnvelope item in list)
{
cntr++;
importSuccessfull = ProcessEnvelope(item, registryEntityType, entityTypeId, cntr, downloadOnly);
if (maxRecords > 0 && cntr >= maxRecords)
{
break;
}
} //end foreach
pageNbr++;
if ((maxRecords > 0 && cntr >= maxRecords))
{
isComplete = true;
LoggingHelper.DoTrace(2, string.Format("Import {2} EARLY EXIT. Completed {0} records out of a total of {1} for {2} ", cntr, pTotalRows, registryEntityType));
}
else if (cntr >= actualTotalRows)
{
isComplete = true;
//LoggingHelper.DoTrace( 2, string.Format( "Completed {0} records out of a total of {1} for {2}", cntr, pTotalRows, registryEntityType ) );
}
//if ( pageNbr * pageSize < pTotalRows )
// pageNbr++;
//else
// isComplete = true;
}
importResults = string.Format("Import {0} - Processed {1} records, with {2} exceptions. \r\n", registryEntityType, cntr, exceptionCtr);
LoggingHelper.DoTrace(2, importResults);
if (!string.IsNullOrWhiteSpace(importNote))
{
importResults += importNote;
}
recordsImported += cntr;
return(importResults);
}
public string Import(string startingDate, string endingDate, int maxRecords, string community, bool downloadOnly = false)
{
bool importingThisType = UtilityManager.GetAppKeyValue("importing_competency_frameworks", true);
if (!importingThisType)
{
LoggingHelper.DoTrace(1, "=== ***************** Skipping import of Competency Frameworks ***************** ");
return("Skipped import of competency_frameworks");
}
LoggingHelper.DoTrace(1, string.Format("=== ***************** {0} ***************** ", thisClassName));
JsonEntity input = new JsonEntity();
ReadEnvelope envelope = new ReadEnvelope();
List <ReadEnvelope> list = new List <ReadEnvelope>();
EntityServices mgr = new EntityServices();
string type = "competency_framework";
int pageNbr = 1;
int pageSize = 75;
string importError = "";
string importResults = "";
string importNote = "";
ThisEntity output = new ThisEntity();
List <string> messages = new List <string>();
//the nbr of records needs to be monitored, to determine the optimum
//NOTE: consider the IOER approach that all candidate records are first downloaded, and then a separate process does the import
int cntr = 0;
int pTotalRows = 0;
int exceptionCtr = 0;
string statusMessage = "";
bool isComplete = false;
bool importSuccessfull = true;
int newImportId = 0;
SaveStatus status = new SaveStatus();
//will need to handle multiple calls - watch for time outs
while (pageNbr > 0 && !isComplete)
{
list = RegistryServices.Search(type, startingDate, endingDate, pageNbr, pageSize, ref pTotalRows, ref statusMessage, community);
//list = GetLatest( type, startingDate, endingDate, pageNbr, pageSize, ref pTotalRows, ref statusMessage, community );
if (list == null || list.Count == 0)
{
isComplete = true;
if (pageNbr == 1)
{
importNote = "Competency Frameworks: No records where found for date range ";
//Console.WriteLine( thisClassName + importNote );
LoggingHelper.DoTrace(4, thisClassName + importNote);
}
break;
}
foreach (ReadEnvelope item in list)
{
cntr++;
string payload = item.DecodedResource.ToString();
LoggingHelper.DoTrace(2, string.Format("{0}. EnvelopeIdentifier: {1} ", cntr, item.EnvelopeIdentifier));
status = new SaveStatus();
status.DoingDownloadOnly = downloadOnly;
importError = "";
importSuccessfull = false;
try
{
//Console.WriteLine( string.Format( "{0}. Competency Frameswork Envelope Identifier {1} ", cntr, item.EnvelopeIdentifier ) );
importSuccessfull = entityImportMgr.ProcessEnvelope(mgr, item, status);
}
catch (Exception ex)
{
if (ex.Message.IndexOf("Path '@context', line 1") > 0)
{
importError = "The referenced registry document is using an old schema. Please republish it with the latest schema!";
status.AddError(importError);
}
else
{
LoggingHelper.LogError(ex, string.Format("Exception encountered in envelopeId: {0}", item.EnvelopeIdentifier), false, "CredentialFinder Import exception");
status.AddError(ex.Message);
importError = ex.Message;
LoggingHelper.DoTrace(1, " show exception to use getting related envelopeId " + ex.Message);
}
//make continue on exceptions an option
exceptionCtr++;
if (maxExceptions > 0 && exceptionCtr > maxExceptions)
{
//arbitrarily stop if large number of exceptions
importNote = string.Format(thisClassName + " - Many exceptions ({0}) were encountered during import - abandoning.", exceptionCtr);
Console.WriteLine(importNote);
LoggingHelper.DoTrace(1, importNote);
LoggingHelper.LogError(importNote, true, thisClassName + "- many exceptions");
isComplete = true;
break;
}
}
finally
{
if (!importSuccessfull)
{
if (string.IsNullOrWhiteSpace(importError))
{
importError = string.Join("\r\n", status.GetAllMessages().ToArray());
}
}
//store document
//add indicator of success
newImportId = importMgr.Add(item, entityTypeId, status.Ctid, importSuccessfull, importError, ref messages);
if (newImportId > 0 && status.Messages != null && status.Messages.Count > 0)
{
importMgr.AddMessages(newImportId, status, ref messages);
}
} //finally
if (maxRecords > 0 && cntr > maxRecords)
{
break;
}
} //end foreach
pageNbr++;
if ((maxRecords > 0 && cntr > maxRecords) || cntr == pTotalRows)
{
isComplete = true;
if (cntr < pTotalRows)
{
LoggingHelper.DoTrace(2, string.Format("CompetencyFramesworkImport EARLY EXIT. Completed {0} records out of a total of {1} ", cntr, pTotalRows));
}
}
} //
importResults = string.Format("{0} - Processed {1} records, with {2} exceptions. \r\n", thisClassName, cntr, exceptionCtr);
if (!string.IsNullOrWhiteSpace(importNote))
{
importResults += importNote;
}
//always call, as deletes are not tracked
if (UtilityManager.GetAppKeyValue("updateCompetencyFrameworkReportTotals", false) == true)
{
mgr.UpdateCompetencyFrameworkReportTotals();
}
return(importResults);
}