private int ProcessManagedItemID(List<metaStoreDefinition> metaStoreDefinitions, Guid managedItemID, string systemName, DataRow managedItemInfo, templateDefinition templateDefinition)
{
int errorCount = 0;
ManagedItemStore managedItemStore = null;
DateTime dateOfChange = (DateTime)managedItemInfo[Data.DataNames._C_DateOfChange];
Guid templateID = (Guid)managedItemInfo[Data.DataNames._C_TemplateID];
List<string> basicTermsToProcess = new List<string>();
foreach (string termName in templateDefinition.basicTermNames)
basicTermsToProcess.Add(termName);
List<string> complexListTermsToProcess = new List<string>();
foreach (string termName in templateDefinition.complexListTermNames)
complexListTermsToProcess.Add(termName);
for (int dsdIndex = 0; dsdIndex < metaStoreDefinitions.Count; dsdIndex++)
{
metaStoreDefinition metaStoreDefinition = metaStoreDefinitions[dsdIndex];
if (metaStoreDefinition.managedItemIDs.Contains(managedItemID))
{
bool newData = true;
if (metaStoreDefinition.dataStoreDefinition.DataStoreConfig.LoadType != LoadType.Full)
if (metaStoreDefinition.dataStoreDefinition.LastRunDate.HasValue)
newData = dateOfChange >= metaStoreDefinition.dataStoreDefinition.LastRunDate.Value.AddDays(-1 * metaStoreDefinition.dataStoreDefinition.DataStoreConfig.NoOfDays);
if (newData)
{
if (metaStoreDefinition.templateStores.ContainsKey(templateID))
{
TemplateStore templateStore = metaStoreDefinition.templateStores[templateID];
//Process the one ManagedItemID
if (managedItemStore == null)
{
try
{
managedItemStore = ManagedItem.CreateStore(_log, templateStore.Template,
(string)managedItemInfo[Data.DataNames._C_TemplateDef],
managedItemID,
((string)managedItemInfo[Data.DataNames._C_ManagedItemNumber]).Trim(),
(string)managedItemInfo[Data.DataNames._C_Status],
(string)managedItemInfo[Data.DataNames._C_State],
templateStore.TemplateTerms,
basicTermsToProcess,
complexListTermsToProcess);
}
catch (Exception ex)
{
_log.Error(string.Format("ManagedItem.CreateStore halted for '{0}'. Exception: '{1}'. Stack Trace: '{2}'.", managedItemStore.ManagedItemNumber, ex.Message, ex.StackTrace));
//Mark this managedItemID as completed.
metaStoreDefinition.managedItemIDs.Remove(managedItemID);
errorCount++;
continue;
}
}
//Perform data processing here.
int maxRowCount = managedItemStore.GetRowCount(templateID, templateStore.ReportedTermNames);
List<string> orderedDSFNames = new List<string>();
if (metaStoreDefinition.requiresHeaderRow && !metaStoreDefinition.headerRowWritten)
{
ManagedItemStore.dataRow dataRow = managedItemStore.GetDataRow(true, templateStore, 0, metaStoreDefinition.orderedDSFNames);
metaStoreDefinition.swOutput.WriteLine(dataRow.rowData);
metaStoreDefinition.headerRowWritten = true;
}
for (int rowIndex = 0; rowIndex < maxRowCount; rowIndex++)
{
ManagedItemStore.dataRow dataRow = managedItemStore.GetDataRow(false, templateStore, rowIndex, metaStoreDefinition.orderedDSFNames);
metaStoreDefinition.swOutput.WriteLine(dataRow.rowData);
metaStoreDefinition.outputRowCount = metaStoreDefinition.outputRowCount + 1;
if (dataRow.rowErrors.Count > 0)
metaStoreDefinition.dataRows.Add(dataRow);
}
}
else
{
throw new Exception(string.Format("systemDataStoreDefinition.templateStores is missing templateID {0}", templateID.ToString()));
}
}
//Mark this managedItemID as completed.
metaStoreDefinition.managedItemIDs.Remove(managedItemID);
//If the last one, close out the file.
if (metaStoreDefinition.managedItemIDs.Count == 0)
{
CloseOutMetaStoreDefinition(metaStoreDefinition, systemName);
metaStoreDefinition.closedOut = true;
}
}
metaStoreDefinitions[dsdIndex] = metaStoreDefinition;
}
return errorCount;
}
public int ProcessSystem(Guid systemID, string systemName)
{
int errorCount = 0;
DateTime runDate = DateTime.Now;
DataTable dtDSDs = DataStoreDefinition.GetDataStoreDefinitionsBySystemID(systemID, true);
List<metaStoreDefinition> metaStoreDefinitions = new List<metaStoreDefinition>();
Dictionary<Guid, Dictionary<Guid, string>> termNameLookup = null;
Dictionary<Guid /*TemplateID*/, Dictionary<Guid /*ComplexListID*/, Dictionary<Guid /*FieldID*/, string /*FieldName*/>>> fieldNameLookup = null;
GetTermAndFieldNameLookup(systemID, ref termNameLookup, ref fieldNameLookup);
foreach (DataRow rowDSD in dtDSDs.Rows)
{
metaStoreDefinitions.Add(InitializeMetaStoreDefinition((Guid)rowDSD[Data.DataNames._C_DataStoreDefinitionID], runDate, systemID, termNameLookup, fieldNameLookup));
}
if (metaStoreDefinitions.Count > 0)
{
List<Guid> systemManagedItemIDs = new List<Guid>();
foreach (metaStoreDefinition msd in metaStoreDefinitions)
{
foreach (Guid managedItemID in msd.managedItemIDs)
if (!systemManagedItemIDs.Contains(managedItemID))
systemManagedItemIDs.Add(managedItemID);
}
_log.Info(string.Format("For system '{0}', found {1:D} active DataStoreDefinition(s) and {2:D} unique ManagedItems from the search(es).", systemName, metaStoreDefinitions.Count, systemManagedItemIDs.Count));
List<Guid> skippedManagedItemIDs = new List<Guid>();
//Create a collection of all the basicTerms, Complex List Terms, that will be needed for all the DSD's.
Dictionary<Guid, templateDefinition> systemTemplateDefinitions = new Dictionary<Guid, templateDefinition>();
foreach (metaStoreDefinition msd in metaStoreDefinitions)
{
foreach (KeyValuePair<Guid /*TemplateID*/, TemplateStore> templateStoreDict in msd.templateStores)
{
templateDefinition systemTemplateDefinition;
if (!systemTemplateDefinitions.ContainsKey(templateStoreDict.Key))
{
systemTemplateDefinition = new templateDefinition();
systemTemplateDefinition.basicTermNames = new List<string>();
systemTemplateDefinition.complexListTermNames = new List<string>();
systemTemplateDefinitions.Add(templateStoreDict.Key, systemTemplateDefinition);
}
else
{
systemTemplateDefinition = systemTemplateDefinitions[templateStoreDict.Key];
}
//Add unique term names here for templateDefinition.
foreach (KeyValuePair<string /*Term Name*/, Term> termDict in templateStoreDict.Value.TemplateTerms)
{
DataStoreField dsf = templateStoreDict.Value.DataStoreFields.FirstOrDefault(kvp => kvp.Value.TermName.Equals(termDict.Key)).Value;
if (dsf != null)
{
switch (termDict.Value.TermType)
{
case TermType.ComplexList:
if (!systemTemplateDefinition.complexListTermNames.Contains(termDict.Key))
systemTemplateDefinition.complexListTermNames.Add(termDict.Key);
break;
default:
if (!systemTemplateDefinition.basicTermNames.Contains(termDict.Key))
systemTemplateDefinition.basicTermNames.Add(termDict.Key);
break;
}
}
}
}
}
//Note - Temporary metrics info here...
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
int miCount = 0;
int basicTermsCount = 0;
int complexListTermsCount = 0;
while (true)
{
Guid? managedItemID = GetNextManagedItemID(metaStoreDefinitions);
if (managedItemID.HasValue)
{
try
{
DataTable dtManagedItemInfo = Business.ManagedItem.GetRawManagedItem(managedItemID.Value);
if (dtManagedItemInfo.Rows.Count > 0)
{
DataRow managedItemInfo = dtManagedItemInfo.Rows[0];
Guid templateID = (Guid)managedItemInfo[Data.DataNames._C_TemplateID];
if (systemTemplateDefinitions.ContainsKey(templateID))
{
basicTermsCount = systemTemplateDefinitions[templateID].basicTermNames.Count;
complexListTermsCount = systemTemplateDefinitions[templateID].complexListTermNames.Count;
errorCount += ProcessManagedItemID(metaStoreDefinitions, managedItemID.Value, systemName, managedItemInfo, systemTemplateDefinitions[templateID]);
miCount++;
}
else
{
SkipManagedItemID(metaStoreDefinitions, managedItemID.Value, skippedManagedItemIDs);
_log.Error(string.Format("When processing managedItemID {0}, systemTemplateDefinitions has {1:D} members and did not contain data for templateID '{2}'", managedItemID.Value.ToString(), systemTemplateDefinitions.Count, templateID.ToString()));
errorCount++;
}
}
else
{
SkipManagedItemID(metaStoreDefinitions, managedItemID.Value, skippedManagedItemIDs);
_log.Error(string.Format("GetRawManagedItem call failed for ManagedItemID '{0}'. The item was not processed.", managedItemID.Value));
errorCount++;
}
}
catch (Exception ex)
{
SkipManagedItemID(metaStoreDefinitions, managedItemID.Value, skippedManagedItemIDs);
_log.Error(string.Format("ProcessManagedItemID halted for '{0}'. Exception: '{1}'. Stack Trace: '{2}'.", managedItemID.Value, ex.Message, ex.StackTrace));
errorCount++;
continue;
}
if (miCount > _HI_MANAGEDITEM_COUNT)
{
_log.Error(string.Format("ProcessSystem halted for system '{0}' due to a high ManagedItem count of '{1:D}' with ManagedItemID = '{2}'.", systemName, miCount, managedItemID.Value));
errorCount++;
break;
}
}
else
{
break;
}
}
stopWatch.Stop();
double tsAll = (double)stopWatch.ElapsedMilliseconds;
double avgMS = 0;
if (miCount > 0)
avgMS = tsAll / miCount;
_log.Info(string.Format("{0} ManagedItems were processed in {1} milliseconds ({2:D2}:{3:D2} minutes) - average = {4:F4} ms. {5} Basic Terms, {6} ComplexList Terms",
miCount.ToString(),
tsAll.ToString(),
stopWatch.Elapsed.Minutes,
stopWatch.Elapsed.Seconds,
avgMS,
basicTermsCount.ToString(),
complexListTermsCount.ToString()));
if (skippedManagedItemIDs.Count > 0)
{
_log.Info("The following ManagedItems were skipped:");
List<string> managedItemIds = new List<string>();
foreach (Guid managedItemID in skippedManagedItemIDs)
{
if (managedItemIds.Count < 50)
managedItemIds.Add(managedItemID.ToString());
else
{
_log.Info(string.Join(",", managedItemIds.ToArray()));
managedItemIds = new List<string>();
}
}
if (managedItemIds.Count > 0)
_log.Info(string.Join(",", managedItemIds.ToArray()));
}
//Ensure that all of the metaStoreDefinitions are closed out.
for (int index = 0; index < metaStoreDefinitions.Count; index++)
{
metaStoreDefinition msd = metaStoreDefinitions[index];
CloseOutMetaStoreDefinition(msd, systemName);
msd.closedOut = true;
metaStoreDefinitions[index] = msd;
}
}
else
{
_log.Info(string.Format("For system '{0}', no active DataStoreDefinitions found.", systemName));
}
return errorCount;
}