Пример #1
0
        /// <summary>
        ///
        /// This method serves as the main logic of this function, for each Process performing the 4 main steps addressed
        /// in the description of the class.  Those 4 steps can be generalized into just 2 steps, both driven by metadata:
        ///
        /// 1.) The record (i.e., product) data will be pulled down through the REST API in raw form
        /// 2.) The record (i.e., product) data will then be applied (i.e., upserted) to a staging table
        ///
        /// <returns>None.</returns>
        /// </summary>
        private void ConsumeData()
        {
            string sSubject = "AceEngine::ConsumeData()";

            try
            {
                using (AceMetadataFactory MetadataFactory = new AceMetadataFactory(moStgConnectionMetadata))
                {
                    using (AceChangeProcessWriter oProcessWriter = new AceChangeProcessWriter(moStgConnectionMetadata))
                    {
                        using (AceChangeRecordWriter oProductWriter = new AceChangeRecordWriter(moStgConnectionMetadata))
                        {
                            List <AceProcess> CurrentJobs = MetadataFactory.GetActiveJobs();

                            foreach (AceProcess TempProcess in CurrentJobs)
                            {
                                LogInfo(sSubject, "Processing Job [" + TempProcess.ProcessID + "] : (" + TempProcess.ProcessName + ")");
                                System.Console.Out.Flush();

                                PullDataSnapshot(oProcessWriter, oProductWriter, TempProcess);
                                System.Console.Out.Flush();

                                ApplyDataSnapshot(TempProcess);
                                System.Console.Out.Flush();
                            }
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                LogError(sSubject, "ERROR!  An error has taken place with record", ex);
            }
        }
Пример #2
0
        /// <summary>
        ///
        /// According to the direction of the metadata of the configured process, this method will retrieve data through
        /// a specified REST API in regard to a particular list of identified records.
        ///
        /// <param name="poProcessWriter">The writer that will record the progress of this instance for the configured Process (represented by 'poProcess')</param>
        /// <param name="poRecordWriter">The writer that will record the raw payload for each record retrieved in this run</param>
        /// <param name="poProcess">The structure that represents the Process being currently run</param>
        /// <returns>The ChangeSeq (i.e., PID) assigned to this particular instance of the Process being run</returns>
        /// </summary>
        private long PullDataForHardCodedKeys(AceChangeProcessWriter poProcessWriter, AceChangeRecordWriter poRecordWriter, AceProcess poProcess)
        {
            long   nTmpKey        = -1;
            string sTmpChangeBody = "";
            string sTmpDataBody   = "";
            string sInfoMsg       = "";
            string sErrMsg        = "";
            string sSubject       = "AceEngine::PullDataForHardCodedKeys()";

            Dictionary <string, string> oTmpFilterArgs = new Dictionary <string, string>();

            if ((poProcess.ChangeAPIConfiguration.KeyList == null) || (poProcess.ChangeAPIConfiguration.KeyList.Count <= 0))
            {
                throw new Exception("ERROR!  No expected items found in the hard-coded key list.");
            }

            poProcess.ChangeSeq = poProcessWriter.InsertProcessInstance(poProcess.ProcessID);

            foreach (string sTmpKey in poProcess.ChangeAPIConfiguration.KeyList)
            {
                // The format for this change manifest request can be hard-coded (as it is here) or it could be a part of the configurable metadata
                sTmpChangeBody = String.Format(AceXmlReader.CONST_DEFAULT_CHG_MANIFEST_REQUEST_XML_BODY, sTmpKey);

                try
                {
                    nTmpKey = Convert.ToInt64(sTmpKey);
                }
                catch (Exception ex)
                {
                    LogError(sSubject, "ERROR!  Could not convert EAN (" + sTmpKey + ") to a number", ex);
                }

                oTmpFilterArgs = AceXmlReader.ExtractFilterArgs(sTmpChangeBody, poProcess.DataAPIConfiguration.RequestFilterArgs);
                sTmpDataBody   = AceXmlReader.PullData(poProcess.DataAPIConfiguration.BaseURL, oTmpFilterArgs, poProcess.DataAPIConfiguration.RequestHeaderArgs);

                if (!String.IsNullOrEmpty(sTmpKey))
                {
                    poRecordWriter.InsertProductInstance(poProcess.ChangeSeq, nTmpKey, sTmpChangeBody, sTmpDataBody);
                }
                else
                {
                    LogError(sSubject, "ERROR!  Provided key was null");
                }
            }

            return(poProcess.ChangeSeq);
        }
Пример #3
0
        /// <summary>
        ///
        /// According to the direction of the metadata for each configured process, this method will then determine the next actions
        /// that should be taken, using various state data (like the status of the last run) to make a determination.  Then, it will
        /// retrieve data through a specified REST API and then persist the returned raw payloads into a table for later usage.
        ///
        /// <param name="poProcessWriter">The writer that will record the progress of this instance for the configured Process (represented by 'poTempProcess')</param>
        /// <param name="poProductWriter">The writer that will record the raw payload for each record retrieved in this run</param>
        /// <param name="poTempProcess">The structure that represents the Process being currently run</param>
        /// <returns>The ChangeSeq (i.e., PID) assigned to this particular instance of the Process being run</returns>
        /// </summary>
        private long PullDataSnapshot(AceChangeProcessWriter poProcessWriter, AceChangeRecordWriter poProductWriter, AceProcess poTempProcess)
        {
            int    nTotalRecords = 0;
            string sSubject      = "AceEngine::PullDataSnapshot()";

            StringBuilder sbLastAnchor = new StringBuilder();

            Dictionary <string, string> oTmpFilterArgs = new Dictionary <string, string>();

            // If there is no change URL specified, we can assume that the records are retrieved as either a hard-coded list or a full set
            if (String.IsNullOrEmpty(poTempProcess.ChangeAPIConfiguration.BaseURL.Trim()))
            {
                long nChangeSeq = -1;

                try
                {
                    nTotalRecords = poTempProcess.ChangeAPIConfiguration.KeyList.Count;

                    nChangeSeq = PullDataForHardCodedKeys(poProcessWriter, poProductWriter, poTempProcess);
                    return(nChangeSeq);
                }
                finally
                {
                    if (nChangeSeq > 0)
                    {
                        poProcessWriter.SetProcessComplete(poTempProcess.ChangeSeq, poTempProcess.ProcessID, nTotalRecords);
                    }
                }
            }

            // If there is a change URL specified, we process normally by acquiring the delta manifest and then retrieving the delta records
            if ((poTempProcess.ChangeSeq = poProcessWriter.DetectPreviousFailure(poTempProcess.ProcessID, sbLastAnchor)) > 0)
            {
                string sLastAnchor = sbLastAnchor.ToString();

                if (!String.IsNullOrEmpty(sLastAnchor))
                {
                    poTempProcess.ChangeAPIConfiguration.CurrentAnchor = sLastAnchor;
                }
            }
            else
            {
                poTempProcess.ChangeSeq = poProcessWriter.InsertProcessInstance(poTempProcess.ProcessID);
            }

            // Indicates that the first pull of the delta data has not yet happened
            if (String.IsNullOrEmpty(poTempProcess.ChangeAPIConfiguration.CurrentAnchor))
            {
                sbLastAnchor = new StringBuilder();

                DateTime oMaxStartDtime = poProcessWriter.GetMaxStartDtime(poTempProcess.ProcessID, sbLastAnchor);

                if (!String.IsNullOrEmpty(poTempProcess.ChangeAPIConfiguration.SinceURLArg))
                {
                    string sSinceURLArg = poTempProcess.ChangeAPIConfiguration.SinceURLArg;

                    if (sbLastAnchor.Length <= 0)
                    {
                        poTempProcess.ChangeAPIConfiguration.CurrentAnchor = sbLastAnchor.ToString();
                    }
                    else if (poTempProcess.ChangeAPIConfiguration.RequestFilterArgs.ContainsKey(sSinceURLArg))
                    {
                        string sSinceValue = poTempProcess.ChangeAPIConfiguration.RequestFilterArgs[sSinceURLArg];
                        if (sSinceValue.EndsWith("d"))
                        {
                            int      nDays       = Convert.ToInt32(sSinceValue.Remove(sSinceValue.IndexOf('d')));
                            DateTime oSinceDtime = DateTime.Now.Subtract(new TimeSpan(nDays, 0, 0, 0, 0));

                            // For the Epoch calculation, our local time needs to be converted to universal time (i.e., GMT)
                            TimeSpan epochTimespan           = oSinceDtime.ToUniversalTime() - new DateTime(1970, 1, 1);
                            long     nMillisecondsSinceEpoch = (long)(epochTimespan.TotalSeconds * 1000);

                            poTempProcess.ChangeAPIConfiguration.RequestFilterArgs[sSinceURLArg] = Convert.ToString(nMillisecondsSinceEpoch);
                        }
                    }
                    else
                    {
                        // For the Epoch calculation, our local time needs to be converted to universal time (i.e., GMT)
                        DateTime tenMinutesEarlier       = oMaxStartDtime.Subtract(new TimeSpan(0, 0, 15, 0, 0));
                        TimeSpan epochTimespan           = tenMinutesEarlier.ToUniversalTime() - new DateTime(1970, 1, 1);
                        long     nMillisecondsSinceEpoch = (long)(epochTimespan.TotalSeconds * 1000);

                        poTempProcess.ChangeAPIConfiguration.RequestFilterArgs[sSinceURLArg] = Convert.ToString(nMillisecondsSinceEpoch);
                    }
                }
            }

            poProcessWriter.CurrentProcessID = poTempProcess.ProcessID;
            poProcessWriter.CurrentChangeSeq = poTempProcess.ChangeSeq;

            if (EnumerateApiAndPersistRawData(poProcessWriter, poProductWriter, poTempProcess))
            {
                LogInfo(sSubject, "Setting the Change ID [" + poTempProcess.ChangeSeq + "] to a success!");
                poProcessWriter.SetProcessComplete(poTempProcess.ChangeSeq, poTempProcess.ProcessID, nTotalRecords);
            }
            else
            {
                LogInfo(sSubject, "Setting the Change ID [" + poTempProcess.ChangeSeq + "] to a failure!");
                poProcessWriter.SetProcessFailure(poTempProcess.ChangeSeq, poTempProcess.ProcessID, poTempProcess.ChangeAPIConfiguration.CurrentAnchor);
            }

            return(poTempProcess.ChangeSeq);
        }
Пример #4
0
        /// <summary>
        ///
        /// This method will do the actual work of enumerating through the REST API and making the web requests for data,
        /// both the change manifests that direct calls and/or the actual data.
        ///
        /// <param name="poProcessWriter">The writer that will record the progress of this instance for the configured Process (represented by 'poTempProcess')</param>
        /// <param name="poProductWriter">The writer that will record the raw payload for each record retrieved in this run</param>
        /// <param name="poTempProcess">The structure that represents the Process being currently run</param>
        /// <returns>The boolean that indicates whether or not the enumeration and persistence succeeded</returns>
        /// </summary>
        private bool EnumerateApiAndPersistRawData(AceChangeProcessWriter poProcessWriter, AceChangeRecordWriter poProductWriter, AceProcess poTempProcess)
        {
            bool   bSuccess       = true;
            int    nTotalRecords  = 0;
            long   nTmpKey        = 0;
            string sTmpKey        = "";
            string sTmpChangeBody = "";
            string sTmpDataBody   = "";
            string sSubject       = "AceEngine::EnumerateApiAndPersistRawData()";
            string sErrMsg        = "";

            Dictionary <string, string> oTmpFilterArgs = new Dictionary <string, string>();

            using (AceXmlReader oChangeDataReader = new AceXmlReader(poTempProcess.ChangeAPIConfiguration))
            {
                oChangeDataReader.FoundNewAnchorCallback = poProcessWriter.UpsertAnchor;

                try
                {
                    foreach (Hashtable oTmpRecord in oChangeDataReader)
                    {
                        try
                        {
                            sTmpKey        = (string)oTmpRecord[poTempProcess.ChangeAPIConfiguration.TargetKeyTag];
                            sTmpChangeBody = (string)oTmpRecord[AceXmlReader.CONST_RESPONSE_XML_BODY_KEY];

                            try
                            {
                                nTmpKey = Convert.ToInt64(sTmpKey);
                            }
                            catch (Exception ex)
                            {
                                LogError(sSubject, "ERROR!  Could not convert EAN (" + sTmpKey + ") to a number", ex);
                            }

                            if (!String.IsNullOrEmpty(sTmpKey))
                            {
                                oTmpFilterArgs = AceXmlReader.ExtractFilterArgs(sTmpChangeBody, poTempProcess.DataAPIConfiguration.RequestFilterArgs);

                                try
                                {
                                    sTmpDataBody =
                                        AceXmlReader.PullData(poTempProcess.DataAPIConfiguration.BaseURL,
                                                              oTmpFilterArgs,
                                                              poTempProcess.DataAPIConfiguration.RequestHeaderArgs);
                                }
                                catch (WebException ex)
                                {
                                    sErrMsg = "ERROR!  Web connection issues occurred when handling EAN (" + sTmpKey + ")";

                                    LogError(sSubject, sErrMsg, ex);

                                    sTmpDataBody = CONST_DATA_URL_ISSUE_ERR_MSG;
                                }

                                poProductWriter.InsertProductInstance(poTempProcess.ChangeSeq, nTmpKey, sTmpChangeBody, sTmpDataBody);
                            }
                            else
                            {
                                LogError(sSubject, "ERROR!  A provided key was null.");
                            }
                        }
                        catch (Exception ex)
                        {
                            sErrMsg = "ERROR!  Could not handle product instance for EAN (" + sTmpKey + ")";

                            LogError(sSubject, sErrMsg, ex);
                        }
                        finally
                        {
                            ++nTotalRecords;
                        }

                        if ((nTotalRecords % 1000) == 0)
                        {
                            LogInfo(sSubject, "Pulled (" + nTotalRecords + ") snapshots to the change_product table");
                            Console.Out.Flush();
                        }
                    } // foreach loop
                }
                catch (WebException ex)
                {
                    sErrMsg = "ERROR!  Web connection issues when attempting to get catalog data...pulling snapshot is stopping now.";
                    LogError(sSubject, sErrMsg, ex);

                    bSuccess = false;
                }
                catch (SqlException ex)
                {
                    sErrMsg = "ERROR!  Database issues when attempting to get catalog data...pulling snapshot is stopping now.";
                    LogError(sSubject, sErrMsg, ex);

                    bSuccess = false;
                }
                catch (Exception ex)
                {
                    sErrMsg = "ERROR!  General issue when attempting to get catalog data...pulling snapshot is stopping now.";
                    LogError(sSubject, sErrMsg, ex);

                    bSuccess = false;
                }
            }

            return(bSuccess);
        }