/// <summary>
///
/// This method will extract the detailed information (like the payload's tags mapping to which
/// columns of a staging table) needed in order to drive the retrieval of data through the API.
///
/// <param name="poProcessDetailsReader">The reader that pulls metadata from the configuration tables</param>
/// <param name="poTmpConfig">The structure that will hold all of the extract config metadata</param>
/// <returns>None.</returns>
/// </summary>
private void SetAPIDetails(SqlDataReader poAPIDetailsReader, AceAPIConfiguration poTmpConfig)
{
while (poAPIDetailsReader.Read())
{
string sBucketName = poAPIDetailsReader["bucket_nm"].ToString();
string sTableName = poAPIDetailsReader["target_table_nm"].ToString();
if (!poTmpConfig.ApplyBuckets.Keys.Contains(sBucketName))
{
poTmpConfig.ApplyBuckets[sBucketName] = new AceAPIBucket(sBucketName, sTableName);
}
AceAPIBucket oTmpBucket = poTmpConfig.ApplyBuckets[sBucketName];
if (String.IsNullOrEmpty(oTmpBucket.BucketName))
{
oTmpBucket.BucketName = sBucketName;
oTmpBucket.TableName = sTableName;
}
// Add new row to bucket here
if (!poAPIDetailsReader.IsDBNull(2))
{
string sAttrName = poAPIDetailsReader["attr_nm"].ToString();
string sAttrType = poAPIDetailsReader["attr_ora_type"].ToString();
string sAttrLen = poAPIDetailsReader["attr_ora_type_len"].ToString();
string sAttrIsKey = poAPIDetailsReader["attr_is_key"].ToString();
string sAttrXPath = poAPIDetailsReader["attr_xpath"].ToString();
string sAttrIsXmlBody = poAPIDetailsReader["attr_is_xml_body"].ToString();
int nAttrLen = -1;
if (!String.IsNullOrEmpty(sAttrLen))
{
nAttrLen = Convert.ToInt32(sAttrLen);
}
SqlDbType oOraDbType = SqlDbType.VarChar;
if (!String.IsNullOrEmpty(sAttrType))
{
oOraDbType = GetMappedOraDbType(sAttrType);
}
bool bIsKey = false;
if (!String.IsNullOrEmpty(sAttrIsKey))
{
bIsKey = (sAttrIsKey == "Y") ? true : false;
}
bool bIsXmlBody = false;
if (!String.IsNullOrEmpty(sAttrIsXmlBody))
{
bIsXmlBody = (sAttrIsXmlBody == "Y") ? true : false;
}
oTmpBucket.AddTargetColumn(sAttrName, oOraDbType, bIsKey, nAttrLen, sAttrXPath, bIsXmlBody);
}
}
}
/// <summary>
///
/// This method will populate a provided Hashtable with data retrieved through the provided SqlDataReader.
/// After obtaining the data payload of the target record, it will use the metadata's XPath values to
/// parse the payload and pull out values of interest.
///
/// <param name="poNewProductReader">The Reader that is enumerating through the dataset of payloads (which were pulled via the REST API)</param>
/// <param name="poTmpConfig">The configuration for the currently running Process</param>
/// <param name="poNewProductRecord">The container that will hold the values parsed from the raw data payload for our record (i.e. product)</param>
/// <returns>None</returns>
/// </summary>
static public void PopulateProductData(SqlDataReader poNewProductReader, AceAPIConfiguration poTmpConfig, Hashtable poNewProductRecord)
{
string sDataRecord = poNewProductReader[0].ToString();
string sXPath = null;
XDocument oDataDoc = null;
if (!sDataRecord.Contains("<errors>") && !sDataRecord.Contains("<error>"))
{
using (StringReader oDataReader = new StringReader(sDataRecord))
{
oDataDoc = XDocument.Load(oDataReader, LoadOptions.PreserveWhitespace);
}
foreach (string sTmpBucketName in poTmpConfig.ApplyBuckets.Keys)
{
AceAPIBucket oTempBucket = poTmpConfig.ApplyBuckets[sTmpBucketName];
foreach (string sTmpAttrName in oTempBucket.SoughtColXPaths.Keys)
{
sXPath = oTempBucket.SoughtColXPaths[sTmpAttrName];
try
{
if (oTempBucket.SoughtColXmlBodies.Keys.Contains(sTmpAttrName) && oTempBucket.SoughtColXmlBodies[sTmpAttrName])
{
if (oDataDoc.XPathSelectElement(sXPath) != null)
{
poNewProductRecord[sTmpAttrName] = oDataDoc.XPathSelectElement(sXPath).ToString();
}
}
else if (oDataDoc.XPathSelectElement(sXPath) != null)
{
poNewProductRecord[sTmpAttrName] = oDataDoc.XPathSelectElement(sXPath).Value;
}
}
catch (Exception ex)
{
// Any logging should occur here
throw ex;
}
}
}
}
else
{
// Any logging should occur here
poNewProductRecord["error"] = sDataRecord;
}
}
/// <summary>
///
/// This method will run a query on the ACE_CFG_API table and retrieve all
/// the necessary metadata for API retrieval jobs that are currently active.
///
/// <returns>The IDs of the API retrieval jobs that are currently active</returns>
/// </summary>
public List <AceProcess> GetActiveJobs()
{
List <long> oJobIds = new List <long>();
List <AceProcess> oActiveJobs = new List <AceProcess>();
Dictionary <int, AceProcess> oActiveJobMap = new Dictionary <int, AceProcess>();
if (!ValidateDbConnection())
{
InitDbMembers();
}
oJobIds = GetActiveJobIds();
foreach (int nTmpJobId in oJobIds)
{
string sJobName = "";
string sAPIType = "";
string sChangeURL = "";
string sDataURL = "";
AceProcess oTmpJob = null;
AceAPIConfiguration oTmpConfig = null;
GetProcessDetailsCmd.Parameters[@"pid"].Value = nTmpJobId;
using (SqlDataReader oProcessDetailsReader = GetProcessDetailsCmd.ExecuteReader())
{
oTmpConfig = null;
while (oProcessDetailsReader.Read())
{
if (!oActiveJobMap.Keys.Contains(nTmpJobId))
{
oTmpJob = new AceProcess(nTmpJobId);
oActiveJobMap[nTmpJobId] = oTmpJob;
}
oTmpJob = oActiveJobMap[nTmpJobId];
sAPIType = oProcessDetailsReader[0].ToString();
if (sAPIType == CONST_API_TYPE_CHANGE)
{
oTmpConfig = oTmpJob.ChangeAPIConfiguration = new AceAPIConfiguration();
}
else if (sAPIType == CONST_API_TYPE_DATA)
{
oTmpConfig = oTmpJob.DataAPIConfiguration = new AceAPIConfiguration();
}
if (oTmpConfig != null)
{
SetAPIBasicConfiguration(oProcessDetailsReader, oTmpConfig);
}
}
}
if (oTmpJob.ChangeAPIConfiguration != null)
{
GetAPIDetailsCmd.Parameters[@"pid"].Value = nTmpJobId;
GetAPIDetailsCmd.Parameters[@"at"].Value = CONST_API_TYPE_CHANGE;
using (SqlDataReader oAPIDetailsReader = GetAPIDetailsCmd.ExecuteReader())
{
SetAPIDetails(oAPIDetailsReader, oTmpJob.ChangeAPIConfiguration);
}
}
if (oTmpJob.DataAPIConfiguration != null)
{
GetAPIDetailsCmd.Parameters[@"pid"].Value = nTmpJobId;
GetAPIDetailsCmd.Parameters[@"at"].Value = CONST_API_TYPE_DATA;
using (SqlDataReader oAPIDetailsReader = GetAPIDetailsCmd.ExecuteReader())
{
SetAPIDetails(oAPIDetailsReader, oTmpJob.DataAPIConfiguration);
}
}
}
foreach (int nJobId in oActiveJobMap.Keys)
{
oActiveJobs.Add(oActiveJobMap[nJobId]);
}
return(oActiveJobs);
}
/// <summary>
///
/// This method will extract the general information (like the URLs and their respective arguments)
/// needed in order to drive the retrieval of data through the API.
///
/// <param name="poProcessDetailsReader">The reader that pulls metadata from the configuration tables</param>
/// <param name="poTmpConfig">The structure that will hold all of the extract config metadata</param>
/// <returns>None.</returns>
/// </summary>
private void SetAPIBasicConfiguration(SqlDataReader poProcessDetailsReader, AceAPIConfiguration poTmpConfig)
{
poTmpConfig.BaseURL = poProcessDetailsReader["aca_base_url"].ToString();
poTmpConfig.ApplyBuckets = new Dictionary <string, AceAPIBucket>();
string sBucketList = poProcessDetailsReader["aca_bucket_list"].ToString();
string[] oBucketList = sBucketList.Split(CONST_BUCKET_LIST_DELIM);
foreach (string sBucketName in oBucketList)
{
poTmpConfig.ApplyBuckets[sBucketName] = new AceAPIBucket();
}
poTmpConfig.SinceURLArg = poProcessDetailsReader["aca_since_url_arg_nm"].ToString();
poTmpConfig.AnchorIndicator = poProcessDetailsReader["aca_anchor_ind_tag_nm"].ToString();
poTmpConfig.AnchorElement = poProcessDetailsReader["aca_anchor_val_tag_nm"].ToString();
string sAnchorFilterArgs = poProcessDetailsReader["aca_anchor_filter_args"].ToString();
string[] oAnchorFilterList = sAnchorFilterArgs.Split(CONST_BUCKET_LIST_DELIM);
foreach (string sTmpAnchorFilter in oAnchorFilterList)
{
if (sTmpAnchorFilter.Contains("="))
{
string[] oAnchorFilterPair = sTmpAnchorFilter.Split(CONST_PARAM_VAL_DELIM);
if (oAnchorFilterPair.Length == 2)
{
poTmpConfig.AnchorFilterArgs[oAnchorFilterPair[0]] = oAnchorFilterPair[1];
}
}
}
string sRequestFilterArgs = poProcessDetailsReader["aca_request_filter_args"].ToString();
string[] oRequestFilterList = sRequestFilterArgs.Split(CONST_BUCKET_LIST_DELIM);
foreach (string sTmpRequestFilter in oRequestFilterList)
{
if (sTmpRequestFilter.Contains("="))
{
string[] oRequestFilterPair = sTmpRequestFilter.Split(CONST_PARAM_VAL_DELIM);
if (oRequestFilterPair.Length == 2)
{
poTmpConfig.RequestFilterArgs[oRequestFilterPair[0]] = oRequestFilterPair[1];
}
}
}
poTmpConfig.ResponseFilterPath = poProcessDetailsReader["aca_xpath_resp_filter"].ToString();
poTmpConfig.TargetTag = poProcessDetailsReader["aca_target_chld_tag"].ToString();
poTmpConfig.TargetKeyTag = poProcessDetailsReader["aca_target_chld_key_tag"].ToString();
if (!poProcessDetailsReader.IsDBNull(11))
{
string sKeyList = poProcessDetailsReader["aca_target_key_list"].ToString();
if (sKeyList.Contains("_"))
{
poTmpConfig.KeyList = GetAllKeysFromTable(sKeyList, poTmpConfig.TargetKeyTag);
}
else
{
string[] oKeyList = poProcessDetailsReader["aca_target_key_list"].ToString().Split(CONST_BUCKET_LIST_DELIM);
poTmpConfig.KeyList.UnionWith(oKeyList);
// poTmpConfig.KeyList.InsertRange(0, oKeyList);
}
}
string sContentType = poProcessDetailsReader["aca_content_type"].ToString();
if (sContentType.ToUpper() == "XML")
{
poTmpConfig.DataContentType = ContentType.XML;
}
else if (sContentType.ToUpper() == "JSON")
{
poTmpConfig.DataContentType = ContentType.JSON;
}
else
{
poTmpConfig.DataContentType = ContentType.XML;
}
string sRequestHeaderArgs = poProcessDetailsReader["aca_rq_hdr_args"].ToString();
string[] oRequestHeaderList = sRequestHeaderArgs.Split(CONST_BUCKET_LIST_DELIM);
foreach (string sTmpRequestHeader in oRequestHeaderList)
{
if (sTmpRequestHeader.Contains("="))
{
string[] oRequestHeaderPair = sTmpRequestHeader.Split(CONST_PARAM_VAL_DELIM);
if (oRequestHeaderPair.Length == 2)
{
poTmpConfig.RequestHeaderArgs[oRequestHeaderPair[0]] = oRequestHeaderPair[1];
}
}
}
}
public AceXmlReader(AceAPIConfiguration poConfiguration)
{
APIConfiguration = poConfiguration;
FoundNewAnchorCallback = null;
}
public AceXmlRecordEnumerator(AceXmlReader poXmlReader, AceAPIConfiguration poConfiguration)
{
XmlReader = poXmlReader;
EnumAPIConfiguration = poConfiguration;
}
/// <summary>
///
/// This method will make the next call to the REST API and then retrieve the
/// XML payload as a XDocument.
///
/// <param name="poEnumConfiguration">The Configuration metadata for our calls to the REST API</param>
/// <param name="psRequestURL">The formatted Request URL for our call to the REST API</param>
/// <returns>The XDocument representation of the XML payload from the REST API</returns>
/// </summary>
static public XDocument PullXmlDoc(AceAPIConfiguration poEnumConfiguration, string psRequestURL)
{
bool bContentTypeXml = (poEnumConfiguration.DataContentType == ContentType.XML);
XDocument oXDoc = null;
WebRequest oWebAPIRequest = null;
for (int nRetryCount = 0; nRetryCount < CONST_MAX_RETRY_COUNT; ++nRetryCount)
{
try
{
oWebAPIRequest = WebRequest.Create(psRequestURL);
oWebAPIRequest.Timeout = AceXmlReader.CONST_WEB_REQUEST_TIMEOUT_MS;
// If required by the server, set the credentials.
oWebAPIRequest.Credentials = CredentialCache.DefaultCredentials;
if ((poEnumConfiguration.RequestHeaderArgs != null) && (poEnumConfiguration.RequestHeaderArgs.Count > 0))
{
foreach (string sTmpName in poEnumConfiguration.RequestHeaderArgs.Keys)
{
oWebAPIRequest.Headers.Add(sTmpName, poEnumConfiguration.RequestHeaderArgs[sTmpName]);
}
}
using (WebResponse oWebAPIResponse = oWebAPIRequest.GetResponse())
{
// Get the stream containing content returned by the server.
Stream oDataStream = oWebAPIResponse.GetResponseStream();
// Open the stream using a StreamReader for easy access.
using (StreamReader oWebReader = new StreamReader(oDataStream))
{
if (bContentTypeXml)
{
oXDoc = XDocument.Load(oWebReader, LoadOptions.PreserveWhitespace);
}
else
{
string sJsonOutput = oWebReader.ReadToEnd();
if (sJsonOutput.StartsWith("["))
{
sJsonOutput = "{\n \"root\": { \n \"product\": " + sJsonOutput + "} }";
}
XmlDocument oXmlDoc = (XmlDocument)JsonConvert.DeserializeXmlNode(sJsonOutput);
using (var nodeReader = new XmlNodeReader(oXmlDoc))
{
nodeReader.MoveToContent();
oXDoc = XDocument.Load(nodeReader);
}
}
}
}
break;
}
catch (WebException ex)
{
if ((nRetryCount + 1) < CONST_MAX_RETRY_COUNT)
{
if (oWebAPIRequest != null)
{
System.Console.WriteLine("DEBUG: Timeout (value=" + oWebAPIRequest.Timeout + ") issue with pulling catalog data for URL(" +
oWebAPIRequest.RequestUri.ToString() + ")...attempting to pull the data again..." + DateTime.Now.ToString());
}
Thread.Sleep(5000);
}
else
{
throw ex;
}
}
catch (IOException ex)
{
if ((nRetryCount + 1) < CONST_MAX_RETRY_COUNT)
{
System.Console.WriteLine("DEBUG: General network issue with pulling catalog data for URL(" + oWebAPIRequest.RequestUri.ToString() +
")...attempting to pull the data again...");
Thread.Sleep(5000);
}
else
{
throw ex;
}
}
}
return(oXDoc);
}
/// <summary>
///
/// This method will format a GET request URL for our target REST API. In this case, the request
/// will make an initial call in the enumeration of sought data, using the metadata contained
/// within our configuration.
///
/// <param name="poConfig">The configuration's metadata for the REST API</param>
/// <returns>The formatted call to our intended REST API</returns>
/// </summary>
public static string FormatRequestURL(AceAPIConfiguration poConfig)
{
return(FormatURL(poConfig.BaseURL, poConfig.RequestFilterArgs));
}
/// <summary>
///
/// This method will format a GET request URL for our target REST API. In this case, the request
/// will make a subsequent (i.e., not initial) call in the enumeration of sought data, and it
/// will use an anchor point (record from a previous call) along with various other arguments.
///
/// <param name="poConfig">The configuration's metadata for the REST API</param>
/// <returns>The formatted call to our intended REST API</returns>
/// </summary>
public static string FormatAnchorURL(AceAPIConfiguration poConfig)
{
return(FormatURL(poConfig.BaseURL, poConfig.AnchorFilterArgs));
}
