/// <summary> /// /// This method will extract the detailed information (like the payload's tags mapping to which /// columns of a staging table) needed in order to drive the retrieval of data through the API. /// /// <param name="poProcessDetailsReader">The reader that pulls metadata from the configuration tables</param> /// <param name="poTmpConfig">The structure that will hold all of the extract config metadata</param> /// <returns>None.</returns> /// </summary> private void SetAPIDetails(SqlDataReader poAPIDetailsReader, AceAPIConfiguration poTmpConfig) { while (poAPIDetailsReader.Read()) { string sBucketName = poAPIDetailsReader["bucket_nm"].ToString(); string sTableName = poAPIDetailsReader["target_table_nm"].ToString(); if (!poTmpConfig.ApplyBuckets.Keys.Contains(sBucketName)) { poTmpConfig.ApplyBuckets[sBucketName] = new AceAPIBucket(sBucketName, sTableName); } AceAPIBucket oTmpBucket = poTmpConfig.ApplyBuckets[sBucketName]; if (String.IsNullOrEmpty(oTmpBucket.BucketName)) { oTmpBucket.BucketName = sBucketName; oTmpBucket.TableName = sTableName; } // Add new row to bucket here if (!poAPIDetailsReader.IsDBNull(2)) { string sAttrName = poAPIDetailsReader["attr_nm"].ToString(); string sAttrType = poAPIDetailsReader["attr_ora_type"].ToString(); string sAttrLen = poAPIDetailsReader["attr_ora_type_len"].ToString(); string sAttrIsKey = poAPIDetailsReader["attr_is_key"].ToString(); string sAttrXPath = poAPIDetailsReader["attr_xpath"].ToString(); string sAttrIsXmlBody = poAPIDetailsReader["attr_is_xml_body"].ToString(); int nAttrLen = -1; if (!String.IsNullOrEmpty(sAttrLen)) { nAttrLen = Convert.ToInt32(sAttrLen); } SqlDbType oOraDbType = SqlDbType.VarChar; if (!String.IsNullOrEmpty(sAttrType)) { oOraDbType = GetMappedOraDbType(sAttrType); } bool bIsKey = false; if (!String.IsNullOrEmpty(sAttrIsKey)) { bIsKey = (sAttrIsKey == "Y") ? true : false; } bool bIsXmlBody = false; if (!String.IsNullOrEmpty(sAttrIsXmlBody)) { bIsXmlBody = (sAttrIsXmlBody == "Y") ? true : false; } oTmpBucket.AddTargetColumn(sAttrName, oOraDbType, bIsKey, nAttrLen, sAttrXPath, bIsXmlBody); } } }
/// <summary> /// /// This method will populate a provided Hashtable with data retrieved through the provided SqlDataReader. /// After obtaining the data payload of the target record, it will use the metadata's XPath values to /// parse the payload and pull out values of interest. /// /// <param name="poNewProductReader">The Reader that is enumerating through the dataset of payloads (which were pulled via the REST API)</param> /// <param name="poTmpConfig">The configuration for the currently running Process</param> /// <param name="poNewProductRecord">The container that will hold the values parsed from the raw data payload for our record (i.e. product)</param> /// <returns>None</returns> /// </summary> static public void PopulateProductData(SqlDataReader poNewProductReader, AceAPIConfiguration poTmpConfig, Hashtable poNewProductRecord) { string sDataRecord = poNewProductReader[0].ToString(); string sXPath = null; XDocument oDataDoc = null; if (!sDataRecord.Contains("<errors>") && !sDataRecord.Contains("<error>")) { using (StringReader oDataReader = new StringReader(sDataRecord)) { oDataDoc = XDocument.Load(oDataReader, LoadOptions.PreserveWhitespace); } foreach (string sTmpBucketName in poTmpConfig.ApplyBuckets.Keys) { AceAPIBucket oTempBucket = poTmpConfig.ApplyBuckets[sTmpBucketName]; foreach (string sTmpAttrName in oTempBucket.SoughtColXPaths.Keys) { sXPath = oTempBucket.SoughtColXPaths[sTmpAttrName]; try { if (oTempBucket.SoughtColXmlBodies.Keys.Contains(sTmpAttrName) && oTempBucket.SoughtColXmlBodies[sTmpAttrName]) { if (oDataDoc.XPathSelectElement(sXPath) != null) { poNewProductRecord[sTmpAttrName] = oDataDoc.XPathSelectElement(sXPath).ToString(); } } else if (oDataDoc.XPathSelectElement(sXPath) != null) { poNewProductRecord[sTmpAttrName] = oDataDoc.XPathSelectElement(sXPath).Value; } } catch (Exception ex) { // Any logging should occur here throw ex; } } } } else { // Any logging should occur here poNewProductRecord["error"] = sDataRecord; } }
/// <summary> /// /// This method will run a query on the ACE_CFG_API table and retrieve all /// the necessary metadata for API retrieval jobs that are currently active. /// /// <returns>The IDs of the API retrieval jobs that are currently active</returns> /// </summary> public List <AceProcess> GetActiveJobs() { List <long> oJobIds = new List <long>(); List <AceProcess> oActiveJobs = new List <AceProcess>(); Dictionary <int, AceProcess> oActiveJobMap = new Dictionary <int, AceProcess>(); if (!ValidateDbConnection()) { InitDbMembers(); } oJobIds = GetActiveJobIds(); foreach (int nTmpJobId in oJobIds) { string sJobName = ""; string sAPIType = ""; string sChangeURL = ""; string sDataURL = ""; AceProcess oTmpJob = null; AceAPIConfiguration oTmpConfig = null; GetProcessDetailsCmd.Parameters[@"pid"].Value = nTmpJobId; using (SqlDataReader oProcessDetailsReader = GetProcessDetailsCmd.ExecuteReader()) { oTmpConfig = null; while (oProcessDetailsReader.Read()) { if (!oActiveJobMap.Keys.Contains(nTmpJobId)) { oTmpJob = new AceProcess(nTmpJobId); oActiveJobMap[nTmpJobId] = oTmpJob; } oTmpJob = oActiveJobMap[nTmpJobId]; sAPIType = oProcessDetailsReader[0].ToString(); if (sAPIType == CONST_API_TYPE_CHANGE) { oTmpConfig = oTmpJob.ChangeAPIConfiguration = new AceAPIConfiguration(); } else if (sAPIType == CONST_API_TYPE_DATA) { oTmpConfig = oTmpJob.DataAPIConfiguration = new AceAPIConfiguration(); } if (oTmpConfig != null) { SetAPIBasicConfiguration(oProcessDetailsReader, oTmpConfig); } } } if (oTmpJob.ChangeAPIConfiguration != null) { GetAPIDetailsCmd.Parameters[@"pid"].Value = nTmpJobId; GetAPIDetailsCmd.Parameters[@"at"].Value = CONST_API_TYPE_CHANGE; using (SqlDataReader oAPIDetailsReader = GetAPIDetailsCmd.ExecuteReader()) { SetAPIDetails(oAPIDetailsReader, oTmpJob.ChangeAPIConfiguration); } } if (oTmpJob.DataAPIConfiguration != null) { GetAPIDetailsCmd.Parameters[@"pid"].Value = nTmpJobId; GetAPIDetailsCmd.Parameters[@"at"].Value = CONST_API_TYPE_DATA; using (SqlDataReader oAPIDetailsReader = GetAPIDetailsCmd.ExecuteReader()) { SetAPIDetails(oAPIDetailsReader, oTmpJob.DataAPIConfiguration); } } } foreach (int nJobId in oActiveJobMap.Keys) { oActiveJobs.Add(oActiveJobMap[nJobId]); } return(oActiveJobs); }
/// <summary> /// /// This method will extract the general information (like the URLs and their respective arguments) /// needed in order to drive the retrieval of data through the API. /// /// <param name="poProcessDetailsReader">The reader that pulls metadata from the configuration tables</param> /// <param name="poTmpConfig">The structure that will hold all of the extract config metadata</param> /// <returns>None.</returns> /// </summary> private void SetAPIBasicConfiguration(SqlDataReader poProcessDetailsReader, AceAPIConfiguration poTmpConfig) { poTmpConfig.BaseURL = poProcessDetailsReader["aca_base_url"].ToString(); poTmpConfig.ApplyBuckets = new Dictionary <string, AceAPIBucket>(); string sBucketList = poProcessDetailsReader["aca_bucket_list"].ToString(); string[] oBucketList = sBucketList.Split(CONST_BUCKET_LIST_DELIM); foreach (string sBucketName in oBucketList) { poTmpConfig.ApplyBuckets[sBucketName] = new AceAPIBucket(); } poTmpConfig.SinceURLArg = poProcessDetailsReader["aca_since_url_arg_nm"].ToString(); poTmpConfig.AnchorIndicator = poProcessDetailsReader["aca_anchor_ind_tag_nm"].ToString(); poTmpConfig.AnchorElement = poProcessDetailsReader["aca_anchor_val_tag_nm"].ToString(); string sAnchorFilterArgs = poProcessDetailsReader["aca_anchor_filter_args"].ToString(); string[] oAnchorFilterList = sAnchorFilterArgs.Split(CONST_BUCKET_LIST_DELIM); foreach (string sTmpAnchorFilter in oAnchorFilterList) { if (sTmpAnchorFilter.Contains("=")) { string[] oAnchorFilterPair = sTmpAnchorFilter.Split(CONST_PARAM_VAL_DELIM); if (oAnchorFilterPair.Length == 2) { poTmpConfig.AnchorFilterArgs[oAnchorFilterPair[0]] = oAnchorFilterPair[1]; } } } string sRequestFilterArgs = poProcessDetailsReader["aca_request_filter_args"].ToString(); string[] oRequestFilterList = sRequestFilterArgs.Split(CONST_BUCKET_LIST_DELIM); foreach (string sTmpRequestFilter in oRequestFilterList) { if (sTmpRequestFilter.Contains("=")) { string[] oRequestFilterPair = sTmpRequestFilter.Split(CONST_PARAM_VAL_DELIM); if (oRequestFilterPair.Length == 2) { poTmpConfig.RequestFilterArgs[oRequestFilterPair[0]] = oRequestFilterPair[1]; } } } poTmpConfig.ResponseFilterPath = poProcessDetailsReader["aca_xpath_resp_filter"].ToString(); poTmpConfig.TargetTag = poProcessDetailsReader["aca_target_chld_tag"].ToString(); poTmpConfig.TargetKeyTag = poProcessDetailsReader["aca_target_chld_key_tag"].ToString(); if (!poProcessDetailsReader.IsDBNull(11)) { string sKeyList = poProcessDetailsReader["aca_target_key_list"].ToString(); if (sKeyList.Contains("_")) { poTmpConfig.KeyList = GetAllKeysFromTable(sKeyList, poTmpConfig.TargetKeyTag); } else { string[] oKeyList = poProcessDetailsReader["aca_target_key_list"].ToString().Split(CONST_BUCKET_LIST_DELIM); poTmpConfig.KeyList.UnionWith(oKeyList); // poTmpConfig.KeyList.InsertRange(0, oKeyList); } } string sContentType = poProcessDetailsReader["aca_content_type"].ToString(); if (sContentType.ToUpper() == "XML") { poTmpConfig.DataContentType = ContentType.XML; } else if (sContentType.ToUpper() == "JSON") { poTmpConfig.DataContentType = ContentType.JSON; } else { poTmpConfig.DataContentType = ContentType.XML; } string sRequestHeaderArgs = poProcessDetailsReader["aca_rq_hdr_args"].ToString(); string[] oRequestHeaderList = sRequestHeaderArgs.Split(CONST_BUCKET_LIST_DELIM); foreach (string sTmpRequestHeader in oRequestHeaderList) { if (sTmpRequestHeader.Contains("=")) { string[] oRequestHeaderPair = sTmpRequestHeader.Split(CONST_PARAM_VAL_DELIM); if (oRequestHeaderPair.Length == 2) { poTmpConfig.RequestHeaderArgs[oRequestHeaderPair[0]] = oRequestHeaderPair[1]; } } } }
public AceXmlReader(AceAPIConfiguration poConfiguration) { APIConfiguration = poConfiguration; FoundNewAnchorCallback = null; }
public AceXmlRecordEnumerator(AceXmlReader poXmlReader, AceAPIConfiguration poConfiguration) { XmlReader = poXmlReader; EnumAPIConfiguration = poConfiguration; }
/// <summary> /// /// This method will make the next call to the REST API and then retrieve the /// XML payload as a XDocument. /// /// <param name="poEnumConfiguration">The Configuration metadata for our calls to the REST API</param> /// <param name="psRequestURL">The formatted Request URL for our call to the REST API</param> /// <returns>The XDocument representation of the XML payload from the REST API</returns> /// </summary> static public XDocument PullXmlDoc(AceAPIConfiguration poEnumConfiguration, string psRequestURL) { bool bContentTypeXml = (poEnumConfiguration.DataContentType == ContentType.XML); XDocument oXDoc = null; WebRequest oWebAPIRequest = null; for (int nRetryCount = 0; nRetryCount < CONST_MAX_RETRY_COUNT; ++nRetryCount) { try { oWebAPIRequest = WebRequest.Create(psRequestURL); oWebAPIRequest.Timeout = AceXmlReader.CONST_WEB_REQUEST_TIMEOUT_MS; // If required by the server, set the credentials. oWebAPIRequest.Credentials = CredentialCache.DefaultCredentials; if ((poEnumConfiguration.RequestHeaderArgs != null) && (poEnumConfiguration.RequestHeaderArgs.Count > 0)) { foreach (string sTmpName in poEnumConfiguration.RequestHeaderArgs.Keys) { oWebAPIRequest.Headers.Add(sTmpName, poEnumConfiguration.RequestHeaderArgs[sTmpName]); } } using (WebResponse oWebAPIResponse = oWebAPIRequest.GetResponse()) { // Get the stream containing content returned by the server. Stream oDataStream = oWebAPIResponse.GetResponseStream(); // Open the stream using a StreamReader for easy access. using (StreamReader oWebReader = new StreamReader(oDataStream)) { if (bContentTypeXml) { oXDoc = XDocument.Load(oWebReader, LoadOptions.PreserveWhitespace); } else { string sJsonOutput = oWebReader.ReadToEnd(); if (sJsonOutput.StartsWith("[")) { sJsonOutput = "{\n \"root\": { \n \"product\": " + sJsonOutput + "} }"; } XmlDocument oXmlDoc = (XmlDocument)JsonConvert.DeserializeXmlNode(sJsonOutput); using (var nodeReader = new XmlNodeReader(oXmlDoc)) { nodeReader.MoveToContent(); oXDoc = XDocument.Load(nodeReader); } } } } break; } catch (WebException ex) { if ((nRetryCount + 1) < CONST_MAX_RETRY_COUNT) { if (oWebAPIRequest != null) { System.Console.WriteLine("DEBUG: Timeout (value=" + oWebAPIRequest.Timeout + ") issue with pulling catalog data for URL(" + oWebAPIRequest.RequestUri.ToString() + ")...attempting to pull the data again..." + DateTime.Now.ToString()); } Thread.Sleep(5000); } else { throw ex; } } catch (IOException ex) { if ((nRetryCount + 1) < CONST_MAX_RETRY_COUNT) { System.Console.WriteLine("DEBUG: General network issue with pulling catalog data for URL(" + oWebAPIRequest.RequestUri.ToString() + ")...attempting to pull the data again..."); Thread.Sleep(5000); } else { throw ex; } } } return(oXDoc); }
/// <summary> /// /// This method will format a GET request URL for our target REST API. In this case, the request /// will make an initial call in the enumeration of sought data, using the metadata contained /// within our configuration. /// /// <param name="poConfig">The configuration's metadata for the REST API</param> /// <returns>The formatted call to our intended REST API</returns> /// </summary> public static string FormatRequestURL(AceAPIConfiguration poConfig) { return(FormatURL(poConfig.BaseURL, poConfig.RequestFilterArgs)); }
/// <summary> /// /// This method will format a GET request URL for our target REST API. In this case, the request /// will make a subsequent (i.e., not initial) call in the enumeration of sought data, and it /// will use an anchor point (record from a previous call) along with various other arguments. /// /// <param name="poConfig">The configuration's metadata for the REST API</param> /// <returns>The formatted call to our intended REST API</returns> /// </summary> public static string FormatAnchorURL(AceAPIConfiguration poConfig) { return(FormatURL(poConfig.BaseURL, poConfig.AnchorFilterArgs)); }