Пример #1
0
        private int ProcessManagedItemID(List<metaStoreDefinition> metaStoreDefinitions, Guid managedItemID, string systemName, DataRow managedItemInfo, templateDefinition templateDefinition)
        {
            int errorCount = 0;
            ManagedItemStore managedItemStore = null;
            DateTime dateOfChange = (DateTime)managedItemInfo[Data.DataNames._C_DateOfChange];
            Guid templateID = (Guid)managedItemInfo[Data.DataNames._C_TemplateID];
            List<string> basicTermsToProcess = new List<string>();
            foreach (string termName in templateDefinition.basicTermNames)
                basicTermsToProcess.Add(termName);
            List<string> complexListTermsToProcess = new List<string>();
            foreach (string termName in templateDefinition.complexListTermNames)
                complexListTermsToProcess.Add(termName);

            for (int dsdIndex = 0; dsdIndex < metaStoreDefinitions.Count; dsdIndex++)
            {
                metaStoreDefinition metaStoreDefinition = metaStoreDefinitions[dsdIndex];
                if (metaStoreDefinition.managedItemIDs.Contains(managedItemID))
                {
                    bool newData = true;
                    if (metaStoreDefinition.dataStoreDefinition.DataStoreConfig.LoadType != LoadType.Full)
                        if (metaStoreDefinition.dataStoreDefinition.LastRunDate.HasValue)
                            newData = dateOfChange >= metaStoreDefinition.dataStoreDefinition.LastRunDate.Value.AddDays(-1 * metaStoreDefinition.dataStoreDefinition.DataStoreConfig.NoOfDays);
                    if (newData)
                    {
                        if (metaStoreDefinition.templateStores.ContainsKey(templateID))
                        {
                            TemplateStore templateStore = metaStoreDefinition.templateStores[templateID];
                            //Process the one ManagedItemID
                            if (managedItemStore == null)
                            {
                                try
                                {
                                    managedItemStore = ManagedItem.CreateStore(_log, templateStore.Template,
                                        (string)managedItemInfo[Data.DataNames._C_TemplateDef],
                                        managedItemID,
                                        ((string)managedItemInfo[Data.DataNames._C_ManagedItemNumber]).Trim(),
                                        (string)managedItemInfo[Data.DataNames._C_Status],
                                        (string)managedItemInfo[Data.DataNames._C_State],
                                        templateStore.TemplateTerms,
                                        basicTermsToProcess,
                                        complexListTermsToProcess);
                                }
                                catch (Exception ex)
                                {
                                    _log.Error(string.Format("ManagedItem.CreateStore halted for '{0}'.  Exception: '{1}'.  Stack Trace: '{2}'.", managedItemStore.ManagedItemNumber, ex.Message, ex.StackTrace));
                                    //Mark this managedItemID as completed.
                                    metaStoreDefinition.managedItemIDs.Remove(managedItemID);
                                    errorCount++;
                                    continue;
                                }
                            }

                            //Perform data processing here.
                            int maxRowCount = managedItemStore.GetRowCount(templateID, templateStore.ReportedTermNames);
                            List<string> orderedDSFNames = new List<string>();

                            if (metaStoreDefinition.requiresHeaderRow && !metaStoreDefinition.headerRowWritten)
                            {
                                ManagedItemStore.dataRow dataRow = managedItemStore.GetDataRow(true, templateStore, 0, metaStoreDefinition.orderedDSFNames);
                                metaStoreDefinition.swOutput.WriteLine(dataRow.rowData);
                                metaStoreDefinition.headerRowWritten = true;
                            }

                            for (int rowIndex = 0; rowIndex < maxRowCount; rowIndex++)
                            {
                                ManagedItemStore.dataRow dataRow = managedItemStore.GetDataRow(false, templateStore, rowIndex, metaStoreDefinition.orderedDSFNames);
                                metaStoreDefinition.swOutput.WriteLine(dataRow.rowData);
                                metaStoreDefinition.outputRowCount = metaStoreDefinition.outputRowCount + 1;
                                if (dataRow.rowErrors.Count > 0)
                                    metaStoreDefinition.dataRows.Add(dataRow);
                            }
                        }
                        else
                        {
                            throw new Exception(string.Format("systemDataStoreDefinition.templateStores is missing templateID {0}", templateID.ToString()));
                        }
                    }

                    //Mark this managedItemID as completed.
                    metaStoreDefinition.managedItemIDs.Remove(managedItemID);
                    //If the last one, close out the file.
                    if (metaStoreDefinition.managedItemIDs.Count == 0)
                    {
                        CloseOutMetaStoreDefinition(metaStoreDefinition, systemName);
                        metaStoreDefinition.closedOut = true;
                    }
                }
                metaStoreDefinitions[dsdIndex] = metaStoreDefinition;
            }
            return errorCount;
        }
Пример #2
0
        public int ProcessSystem(Guid systemID, string systemName)
        {
            int errorCount = 0;
            DateTime runDate = DateTime.Now;
            DataTable dtDSDs = DataStoreDefinition.GetDataStoreDefinitionsBySystemID(systemID, true);
            List<metaStoreDefinition> metaStoreDefinitions = new List<metaStoreDefinition>();

            Dictionary<Guid, Dictionary<Guid, string>> termNameLookup = null;
            Dictionary<Guid /*TemplateID*/, Dictionary<Guid /*ComplexListID*/, Dictionary<Guid /*FieldID*/, string /*FieldName*/>>> fieldNameLookup = null;
            GetTermAndFieldNameLookup(systemID, ref termNameLookup, ref fieldNameLookup);

            foreach (DataRow rowDSD in dtDSDs.Rows)
            {
                metaStoreDefinitions.Add(InitializeMetaStoreDefinition((Guid)rowDSD[Data.DataNames._C_DataStoreDefinitionID], runDate, systemID, termNameLookup, fieldNameLookup));
            }

            if (metaStoreDefinitions.Count > 0)
            {
                List<Guid> systemManagedItemIDs = new List<Guid>();
                foreach (metaStoreDefinition msd in metaStoreDefinitions)
                {
                    foreach (Guid managedItemID in msd.managedItemIDs)
                        if (!systemManagedItemIDs.Contains(managedItemID))
                            systemManagedItemIDs.Add(managedItemID);
                }
                _log.Info(string.Format("For system '{0}', found {1:D} active DataStoreDefinition(s) and {2:D} unique ManagedItems from the search(es).", systemName, metaStoreDefinitions.Count, systemManagedItemIDs.Count));

                List<Guid> skippedManagedItemIDs = new List<Guid>();
                //Create a collection of all the basicTerms, Complex List Terms, that will be needed for all the DSD's.
                Dictionary<Guid, templateDefinition> systemTemplateDefinitions = new Dictionary<Guid, templateDefinition>();
                foreach (metaStoreDefinition msd in metaStoreDefinitions)
                {
                    foreach (KeyValuePair<Guid /*TemplateID*/, TemplateStore> templateStoreDict in msd.templateStores)
                    {
                        templateDefinition systemTemplateDefinition;
                        if (!systemTemplateDefinitions.ContainsKey(templateStoreDict.Key))
                        {
                            systemTemplateDefinition = new templateDefinition();
                            systemTemplateDefinition.basicTermNames = new List<string>();
                            systemTemplateDefinition.complexListTermNames = new List<string>();
                            systemTemplateDefinitions.Add(templateStoreDict.Key, systemTemplateDefinition);
                        }
                        else
                        {
                            systemTemplateDefinition = systemTemplateDefinitions[templateStoreDict.Key];
                        }

                        //Add unique term names here for templateDefinition.
                        foreach (KeyValuePair<string /*Term Name*/, Term> termDict in templateStoreDict.Value.TemplateTerms)
                        {
                            DataStoreField dsf = templateStoreDict.Value.DataStoreFields.FirstOrDefault(kvp => kvp.Value.TermName.Equals(termDict.Key)).Value;
                            if (dsf != null)
                            {
                                switch (termDict.Value.TermType)
                                {
                                    case TermType.ComplexList:
                                        if (!systemTemplateDefinition.complexListTermNames.Contains(termDict.Key))
                                            systemTemplateDefinition.complexListTermNames.Add(termDict.Key);
                                        break;

                                    default:
                                        if (!systemTemplateDefinition.basicTermNames.Contains(termDict.Key))
                                            systemTemplateDefinition.basicTermNames.Add(termDict.Key);
                                        break;
                                }
                            }
                        }
                    }
                }

                //Note - Temporary metrics info here...
                Stopwatch stopWatch = new Stopwatch();
                stopWatch.Start();
                int miCount = 0;
                int basicTermsCount = 0;
                int complexListTermsCount = 0;

                while (true)
                {
                    Guid? managedItemID = GetNextManagedItemID(metaStoreDefinitions);
                    if (managedItemID.HasValue)
                    {
                        try
                        {
                            DataTable dtManagedItemInfo = Business.ManagedItem.GetRawManagedItem(managedItemID.Value);
                            if (dtManagedItemInfo.Rows.Count > 0)
                            {
                                DataRow managedItemInfo = dtManagedItemInfo.Rows[0];
                                Guid templateID = (Guid)managedItemInfo[Data.DataNames._C_TemplateID];
                                if (systemTemplateDefinitions.ContainsKey(templateID))
                                {
                                    basicTermsCount = systemTemplateDefinitions[templateID].basicTermNames.Count;
                                    complexListTermsCount = systemTemplateDefinitions[templateID].complexListTermNames.Count;
                                    errorCount += ProcessManagedItemID(metaStoreDefinitions, managedItemID.Value, systemName, managedItemInfo, systemTemplateDefinitions[templateID]);
                                    miCount++;
                                }
                                else
                                {
                                    SkipManagedItemID(metaStoreDefinitions, managedItemID.Value, skippedManagedItemIDs);
                                    _log.Error(string.Format("When processing managedItemID {0}, systemTemplateDefinitions has {1:D} members and did not contain data for templateID '{2}'", managedItemID.Value.ToString(), systemTemplateDefinitions.Count, templateID.ToString()));
                                    errorCount++;
                                }
                            }
                            else
                            {
                                SkipManagedItemID(metaStoreDefinitions, managedItemID.Value, skippedManagedItemIDs);
                                _log.Error(string.Format("GetRawManagedItem call failed for ManagedItemID '{0}'.  The item was not processed.", managedItemID.Value));
                                errorCount++;
                            }
                        }
                        catch (Exception ex)
                        {
                            SkipManagedItemID(metaStoreDefinitions, managedItemID.Value, skippedManagedItemIDs);
                            _log.Error(string.Format("ProcessManagedItemID halted for '{0}'.  Exception: '{1}'.  Stack Trace: '{2}'.", managedItemID.Value, ex.Message, ex.StackTrace));
                            errorCount++;
                            continue;
                        }
                        if (miCount > _HI_MANAGEDITEM_COUNT)
                        {
                            _log.Error(string.Format("ProcessSystem halted for system '{0}' due to a high ManagedItem count of '{1:D}' with ManagedItemID = '{2}'.", systemName, miCount, managedItemID.Value));
                            errorCount++;
                            break;
                        }
                    }
                    else
                    {
                        break;
                    }
                }
                stopWatch.Stop();
                double tsAll = (double)stopWatch.ElapsedMilliseconds;
                double avgMS = 0;
                if (miCount > 0)
                    avgMS = tsAll / miCount;
                _log.Info(string.Format("{0} ManagedItems were processed in {1} milliseconds ({2:D2}:{3:D2} minutes) - average = {4:F4} ms.  {5} Basic Terms, {6} ComplexList Terms",
                    miCount.ToString(),
                    tsAll.ToString(),
                    stopWatch.Elapsed.Minutes,
                    stopWatch.Elapsed.Seconds,
                    avgMS,
                    basicTermsCount.ToString(),
                    complexListTermsCount.ToString()));
                if (skippedManagedItemIDs.Count > 0)
                {
                    _log.Info("The following ManagedItems were skipped:");
                    List<string> managedItemIds = new List<string>();
                    foreach (Guid managedItemID in skippedManagedItemIDs)
                    {
                        if (managedItemIds.Count < 50)
                            managedItemIds.Add(managedItemID.ToString());
                        else
                        {
                            _log.Info(string.Join(",", managedItemIds.ToArray()));
                            managedItemIds = new List<string>();
                        }
                    }
                    if (managedItemIds.Count > 0)
                        _log.Info(string.Join(",", managedItemIds.ToArray()));
                }

                //Ensure that all of the metaStoreDefinitions are closed out.
                for (int index = 0; index < metaStoreDefinitions.Count; index++)
                {
                    metaStoreDefinition msd = metaStoreDefinitions[index];
                    CloseOutMetaStoreDefinition(msd, systemName);
                    msd.closedOut = true;
                    metaStoreDefinitions[index] = msd;
                }
            }
            else
            {
                _log.Info(string.Format("For system '{0}', no active DataStoreDefinitions found.", systemName));
            }
            return errorCount;
        }