} // method() /// <summary> /// /// Using the metadata for the configured Process, this method will determinen whether or not any values /// for a designated Bucket are present in the current record parsed from the raw data payload. /// /// <param name="poProcess">The structure that represents the Process being currently run</param> /// <param name="psBucketName">The logical Bucket currently being targeted by the running instance of the Process</param> /// <param name="poRecord">The current record parsed from the raw data payload and which will be applied to the Bucket's target table/columns</param> /// <returns>A boolean indicating whether or not the record has any relevant values that belong to the logical Bucket</returns> /// </summary> private bool AreBucketValuesPresent(AceProcess poProcess, string psBucketName, Hashtable poRecord) { bool bValuesPresent = false; List <string> oKeys = null; AceAPIBucket oBucket = null; if (!poProcess.DataAPIConfiguration.ApplyBuckets.ContainsKey(psBucketName)) { return(false); } oBucket = poProcess.DataAPIConfiguration.ApplyBuckets[psBucketName]; foreach (string sTmpAttrName in oBucket.SoughtColumns.Keys) { if (!oBucket.ColKeys.Contains(sTmpAttrName)) { string sTmpVal = (string)poRecord[sTmpAttrName]; if (!String.IsNullOrEmpty(sTmpVal)) { bValuesPresent = true; break; } } } return(bValuesPresent); } // method()
/// <summary> /// /// After the enumeration of the REST API is complete, this method will enumerate through the raw payloads that /// were retrieved; it will then use metadata in order to parse and then load the data into specific table columns. /// /// <param name="poProcess">The structure that represents the Process being currently run</param> /// <returns>None</returns> /// </summary> private void ApplyDataSnapshot(AceProcess poProcess) { int nTotalRecords = 0; int nFailedRecords = 0; string sSubject = "PmdAceConsumptionServiceImpl::ApplyDataSnapshot()"; Hashtable CurrRecord = new Hashtable(); try { Dictionary <string, IApplicable> BucketApplyManagers = CreateApplyManagers(poProcess); using (AceChangeRecordReader oRecordReader = new AceChangeRecordReader(moStgConnectionMetadata, poProcess)) { foreach (Hashtable TempRecord in oRecordReader) { CurrRecord = TempRecord; // We should probably have a better way of detecting successful records vs. error records if (TempRecord.Count > 1) { foreach (string sTmpBucketName in BucketApplyManagers.Keys) { if (AreBucketValuesPresent(poProcess, sTmpBucketName, TempRecord)) { IApplicable ApplyManager = BucketApplyManagers[sTmpBucketName]; ApplyManager.UpsertRecord(TempRecord); } } } else { nFailedRecords++; } nTotalRecords++; if ((nTotalRecords % 1000) == 0) { LogInfo(sSubject, "Applied (" + nTotalRecords + ") records to the staging table(s)"); Console.Out.Flush(); } } } } catch (Exception ex) { LogError(sSubject, "ERROR! An error has taken place with record -> Contents: (" + CurrRecord.ToString() + ")", ex); } } // method()
public AceChangeRecordReader(AceConnectionMetadata poDbConnMetadata, AceProcess poAceProcess) { DbConnMetadata = poDbConnMetadata; AceProcess = poAceProcess; DbConnection = null; RetrieveProducts = null; ProductReader = null; InitDBMembers(); StartEnumerator(); }
} // method() /// <summary> /// /// Using the metadata of the designated process, this method will create a mapping between the logical buckets /// of a raw data payload (like a composite XML body of a document) to a table and its various columns. /// /// <param name="poProcess">The structure that represents the Process being currently run</param> /// <returns>None</returns> /// </summary> private Dictionary <string, IApplicable> CreateApplyManagers(AceProcess poProcess) { Dictionary <string, IApplicable> ApplyManagers = new Dictionary <string, IApplicable>(); foreach (string sBucketName in poProcess.DataAPIConfiguration.ApplyBuckets.Keys) { AceAPIBucket TempBucket = poProcess.DataAPIConfiguration.ApplyBuckets[sBucketName]; AceApplyManager TempApplyManager = new AceApplyManager(moStgConnectionMetadata, TempBucket); ApplyManagers[sBucketName] = TempApplyManager; } return(ApplyManagers); }
/// <summary> /// /// According to the direction of the metadata for each configured process, this method will then determine the next actions /// that should be taken, using various state data (like the status of the last run) to make a determination. Then, it will /// retrieve data through a specified REST API and then persist the returned raw payloads into a table for later usage. /// /// <param name="poProcessWriter">The writer that will record the progress of this instance for the configured Process (represented by 'poTempProcess')</param> /// <param name="poProductWriter">The writer that will record the raw payload for each record retrieved in this run</param> /// <param name="poTempProcess">The structure that represents the Process being currently run</param> /// <returns>The ChangeSeq (i.e., PID) assigned to this particular instance of the Process being run</returns> /// </summary> private long PullDataSnapshot(AceChangeProcessWriter poProcessWriter, AceChangeRecordWriter poProductWriter, AceProcess poTempProcess) { int nTotalRecords = 0; string sSubject = "AceEngine::PullDataSnapshot()"; StringBuilder sbLastAnchor = new StringBuilder(); Dictionary <string, string> oTmpFilterArgs = new Dictionary <string, string>(); // If there is no change URL specified, we can assume that the records are retrieved as either a hard-coded list or a full set if (String.IsNullOrEmpty(poTempProcess.ChangeAPIConfiguration.BaseURL.Trim())) { long nChangeSeq = -1; try { nTotalRecords = poTempProcess.ChangeAPIConfiguration.KeyList.Count; nChangeSeq = PullDataForHardCodedKeys(poProcessWriter, poProductWriter, poTempProcess); return(nChangeSeq); } finally { if (nChangeSeq > 0) { poProcessWriter.SetProcessComplete(poTempProcess.ChangeSeq, poTempProcess.ProcessID, nTotalRecords); } } } // If there is a change URL specified, we process normally by acquiring the delta manifest and then retrieving the delta records if ((poTempProcess.ChangeSeq = poProcessWriter.DetectPreviousFailure(poTempProcess.ProcessID, sbLastAnchor)) > 0) { string sLastAnchor = sbLastAnchor.ToString(); if (!String.IsNullOrEmpty(sLastAnchor)) { poTempProcess.ChangeAPIConfiguration.CurrentAnchor = sLastAnchor; } } else { poTempProcess.ChangeSeq = poProcessWriter.InsertProcessInstance(poTempProcess.ProcessID); } // Indicates that the first pull of the delta data has not yet happened if (String.IsNullOrEmpty(poTempProcess.ChangeAPIConfiguration.CurrentAnchor)) { sbLastAnchor = new StringBuilder(); DateTime oMaxStartDtime = poProcessWriter.GetMaxStartDtime(poTempProcess.ProcessID, sbLastAnchor); if (!String.IsNullOrEmpty(poTempProcess.ChangeAPIConfiguration.SinceURLArg)) { string sSinceURLArg = poTempProcess.ChangeAPIConfiguration.SinceURLArg; if (sbLastAnchor.Length <= 0) { poTempProcess.ChangeAPIConfiguration.CurrentAnchor = sbLastAnchor.ToString(); } else if (poTempProcess.ChangeAPIConfiguration.RequestFilterArgs.ContainsKey(sSinceURLArg)) { string sSinceValue = poTempProcess.ChangeAPIConfiguration.RequestFilterArgs[sSinceURLArg]; if (sSinceValue.EndsWith("d")) { int nDays = Convert.ToInt32(sSinceValue.Remove(sSinceValue.IndexOf('d'))); DateTime oSinceDtime = DateTime.Now.Subtract(new TimeSpan(nDays, 0, 0, 0, 0)); // For the Epoch calculation, our local time needs to be converted to universal time (i.e., GMT) TimeSpan epochTimespan = oSinceDtime.ToUniversalTime() - new DateTime(1970, 1, 1); long nMillisecondsSinceEpoch = (long)(epochTimespan.TotalSeconds * 1000); poTempProcess.ChangeAPIConfiguration.RequestFilterArgs[sSinceURLArg] = Convert.ToString(nMillisecondsSinceEpoch); } } else { // For the Epoch calculation, our local time needs to be converted to universal time (i.e., GMT) DateTime tenMinutesEarlier = oMaxStartDtime.Subtract(new TimeSpan(0, 0, 15, 0, 0)); TimeSpan epochTimespan = tenMinutesEarlier.ToUniversalTime() - new DateTime(1970, 1, 1); long nMillisecondsSinceEpoch = (long)(epochTimespan.TotalSeconds * 1000); poTempProcess.ChangeAPIConfiguration.RequestFilterArgs[sSinceURLArg] = Convert.ToString(nMillisecondsSinceEpoch); } } } poProcessWriter.CurrentProcessID = poTempProcess.ProcessID; poProcessWriter.CurrentChangeSeq = poTempProcess.ChangeSeq; if (EnumerateApiAndPersistRawData(poProcessWriter, poProductWriter, poTempProcess)) { LogInfo(sSubject, "Setting the Change ID [" + poTempProcess.ChangeSeq + "] to a success!"); poProcessWriter.SetProcessComplete(poTempProcess.ChangeSeq, poTempProcess.ProcessID, nTotalRecords); } else { LogInfo(sSubject, "Setting the Change ID [" + poTempProcess.ChangeSeq + "] to a failure!"); poProcessWriter.SetProcessFailure(poTempProcess.ChangeSeq, poTempProcess.ProcessID, poTempProcess.ChangeAPIConfiguration.CurrentAnchor); } return(poTempProcess.ChangeSeq); }
/// <summary> /// /// According to the direction of the metadata of the configured process, this method will retrieve data through /// a specified REST API in regard to a particular list of identified records. /// /// <param name="poProcessWriter">The writer that will record the progress of this instance for the configured Process (represented by 'poProcess')</param> /// <param name="poRecordWriter">The writer that will record the raw payload for each record retrieved in this run</param> /// <param name="poProcess">The structure that represents the Process being currently run</param> /// <returns>The ChangeSeq (i.e., PID) assigned to this particular instance of the Process being run</returns> /// </summary> private long PullDataForHardCodedKeys(AceChangeProcessWriter poProcessWriter, AceChangeRecordWriter poRecordWriter, AceProcess poProcess) { long nTmpKey = -1; string sTmpChangeBody = ""; string sTmpDataBody = ""; string sInfoMsg = ""; string sErrMsg = ""; string sSubject = "AceEngine::PullDataForHardCodedKeys()"; Dictionary <string, string> oTmpFilterArgs = new Dictionary <string, string>(); if ((poProcess.ChangeAPIConfiguration.KeyList == null) || (poProcess.ChangeAPIConfiguration.KeyList.Count <= 0)) { throw new Exception("ERROR! No expected items found in the hard-coded key list."); } poProcess.ChangeSeq = poProcessWriter.InsertProcessInstance(poProcess.ProcessID); foreach (string sTmpKey in poProcess.ChangeAPIConfiguration.KeyList) { // The format for this change manifest request can be hard-coded (as it is here) or it could be a part of the configurable metadata sTmpChangeBody = String.Format(AceXmlReader.CONST_DEFAULT_CHG_MANIFEST_REQUEST_XML_BODY, sTmpKey); try { nTmpKey = Convert.ToInt64(sTmpKey); } catch (Exception ex) { LogError(sSubject, "ERROR! Could not convert EAN (" + sTmpKey + ") to a number", ex); } oTmpFilterArgs = AceXmlReader.ExtractFilterArgs(sTmpChangeBody, poProcess.DataAPIConfiguration.RequestFilterArgs); sTmpDataBody = AceXmlReader.PullData(poProcess.DataAPIConfiguration.BaseURL, oTmpFilterArgs, poProcess.DataAPIConfiguration.RequestHeaderArgs); if (!String.IsNullOrEmpty(sTmpKey)) { poRecordWriter.InsertProductInstance(poProcess.ChangeSeq, nTmpKey, sTmpChangeBody, sTmpDataBody); } else { LogError(sSubject, "ERROR! Provided key was null"); } } return(poProcess.ChangeSeq); }
/// <summary> /// /// This method will do the actual work of enumerating through the REST API and making the web requests for data, /// both the change manifests that direct calls and/or the actual data. /// /// <param name="poProcessWriter">The writer that will record the progress of this instance for the configured Process (represented by 'poTempProcess')</param> /// <param name="poProductWriter">The writer that will record the raw payload for each record retrieved in this run</param> /// <param name="poTempProcess">The structure that represents the Process being currently run</param> /// <returns>The boolean that indicates whether or not the enumeration and persistence succeeded</returns> /// </summary> private bool EnumerateApiAndPersistRawData(AceChangeProcessWriter poProcessWriter, AceChangeRecordWriter poProductWriter, AceProcess poTempProcess) { bool bSuccess = true; int nTotalRecords = 0; long nTmpKey = 0; string sTmpKey = ""; string sTmpChangeBody = ""; string sTmpDataBody = ""; string sSubject = "AceEngine::EnumerateApiAndPersistRawData()"; string sErrMsg = ""; Dictionary <string, string> oTmpFilterArgs = new Dictionary <string, string>(); using (AceXmlReader oChangeDataReader = new AceXmlReader(poTempProcess.ChangeAPIConfiguration)) { oChangeDataReader.FoundNewAnchorCallback = poProcessWriter.UpsertAnchor; try { foreach (Hashtable oTmpRecord in oChangeDataReader) { try { sTmpKey = (string)oTmpRecord[poTempProcess.ChangeAPIConfiguration.TargetKeyTag]; sTmpChangeBody = (string)oTmpRecord[AceXmlReader.CONST_RESPONSE_XML_BODY_KEY]; try { nTmpKey = Convert.ToInt64(sTmpKey); } catch (Exception ex) { LogError(sSubject, "ERROR! Could not convert EAN (" + sTmpKey + ") to a number", ex); } if (!String.IsNullOrEmpty(sTmpKey)) { oTmpFilterArgs = AceXmlReader.ExtractFilterArgs(sTmpChangeBody, poTempProcess.DataAPIConfiguration.RequestFilterArgs); try { sTmpDataBody = AceXmlReader.PullData(poTempProcess.DataAPIConfiguration.BaseURL, oTmpFilterArgs, poTempProcess.DataAPIConfiguration.RequestHeaderArgs); } catch (WebException ex) { sErrMsg = "ERROR! Web connection issues occurred when handling EAN (" + sTmpKey + ")"; LogError(sSubject, sErrMsg, ex); sTmpDataBody = CONST_DATA_URL_ISSUE_ERR_MSG; } poProductWriter.InsertProductInstance(poTempProcess.ChangeSeq, nTmpKey, sTmpChangeBody, sTmpDataBody); } else { LogError(sSubject, "ERROR! A provided key was null."); } } catch (Exception ex) { sErrMsg = "ERROR! Could not handle product instance for EAN (" + sTmpKey + ")"; LogError(sSubject, sErrMsg, ex); } finally { ++nTotalRecords; } if ((nTotalRecords % 1000) == 0) { LogInfo(sSubject, "Pulled (" + nTotalRecords + ") snapshots to the change_product table"); Console.Out.Flush(); } } // foreach loop } catch (WebException ex) { sErrMsg = "ERROR! Web connection issues when attempting to get catalog data...pulling snapshot is stopping now."; LogError(sSubject, sErrMsg, ex); bSuccess = false; } catch (SqlException ex) { sErrMsg = "ERROR! Database issues when attempting to get catalog data...pulling snapshot is stopping now."; LogError(sSubject, sErrMsg, ex); bSuccess = false; } catch (Exception ex) { sErrMsg = "ERROR! General issue when attempting to get catalog data...pulling snapshot is stopping now."; LogError(sSubject, sErrMsg, ex); bSuccess = false; } } return(bSuccess); }
/// <summary> /// /// This method will run a query on the ACE_CFG_API table and retrieve all /// the necessary metadata for API retrieval jobs that are currently active. /// /// <returns>The IDs of the API retrieval jobs that are currently active</returns> /// </summary> public List <AceProcess> GetActiveJobs() { List <long> oJobIds = new List <long>(); List <AceProcess> oActiveJobs = new List <AceProcess>(); Dictionary <int, AceProcess> oActiveJobMap = new Dictionary <int, AceProcess>(); if (!ValidateDbConnection()) { InitDbMembers(); } oJobIds = GetActiveJobIds(); foreach (int nTmpJobId in oJobIds) { string sJobName = ""; string sAPIType = ""; string sChangeURL = ""; string sDataURL = ""; AceProcess oTmpJob = null; AceAPIConfiguration oTmpConfig = null; GetProcessDetailsCmd.Parameters[@"pid"].Value = nTmpJobId; using (SqlDataReader oProcessDetailsReader = GetProcessDetailsCmd.ExecuteReader()) { oTmpConfig = null; while (oProcessDetailsReader.Read()) { if (!oActiveJobMap.Keys.Contains(nTmpJobId)) { oTmpJob = new AceProcess(nTmpJobId); oActiveJobMap[nTmpJobId] = oTmpJob; } oTmpJob = oActiveJobMap[nTmpJobId]; sAPIType = oProcessDetailsReader[0].ToString(); if (sAPIType == CONST_API_TYPE_CHANGE) { oTmpConfig = oTmpJob.ChangeAPIConfiguration = new AceAPIConfiguration(); } else if (sAPIType == CONST_API_TYPE_DATA) { oTmpConfig = oTmpJob.DataAPIConfiguration = new AceAPIConfiguration(); } if (oTmpConfig != null) { SetAPIBasicConfiguration(oProcessDetailsReader, oTmpConfig); } } } if (oTmpJob.ChangeAPIConfiguration != null) { GetAPIDetailsCmd.Parameters[@"pid"].Value = nTmpJobId; GetAPIDetailsCmd.Parameters[@"at"].Value = CONST_API_TYPE_CHANGE; using (SqlDataReader oAPIDetailsReader = GetAPIDetailsCmd.ExecuteReader()) { SetAPIDetails(oAPIDetailsReader, oTmpJob.ChangeAPIConfiguration); } } if (oTmpJob.DataAPIConfiguration != null) { GetAPIDetailsCmd.Parameters[@"pid"].Value = nTmpJobId; GetAPIDetailsCmd.Parameters[@"at"].Value = CONST_API_TYPE_DATA; using (SqlDataReader oAPIDetailsReader = GetAPIDetailsCmd.ExecuteReader()) { SetAPIDetails(oAPIDetailsReader, oTmpJob.DataAPIConfiguration); } } } foreach (int nJobId in oActiveJobMap.Keys) { oActiveJobs.Add(oActiveJobMap[nJobId]); } return(oActiveJobs); }