/// <summary>
/// Gets the data from the url in an OData feed and also loops through each
/// other page that OData returns in a parallel fashion.
/// </summary>
/// <param name="Url">The url of the first page. Should not contain pagesize, $count or $skip</param>
/// <param name="pagesize">The number of entries per page. This should be 250 by default, but it might change based on other specifications</param>
/// <param name="ApiParameters">Changes the behavior of the Api calls</param>
/// <returns>A list of JObjects representing the feed.</returns>
public async Task <List <JObject> > AimsODataFeed(string Url, ApiParametersContainer ApiParameters = null, int pagesize = 250)
{
// get first page
LoggingFunction?.Invoke("Retrieving first page of feed");
var firstPageUrl = Url + (Url.Contains("?") ? "&" : "?") + $"$count=true&pagesize={pagesize}";
var firstPageResponse = await this.GetAsync(firstPageUrl, ApiParameters);
if (!firstPageResponse.IsSuccessful)
{
throw new Exception($"Error retrieving response from {firstPageUrl}: {firstPageResponse.Content}");
}
// get count and data from first page
var feedData = new List <JObject>();
var firstPageJson = (JObject)JsonConvert.DeserializeObject(firstPageResponse.Content);
var count = (int)firstPageJson["@odata.count"];
var numPages = count / pagesize + 1;
LoggingFunction?.Invoke($"Successfully retrieved first page. Number of entries: {count}. Number of pages: {numPages}");
((JArray)firstPageJson["value"]).ToList().ForEach(j => feedData.Add((JObject)j));
// get the pages in an asynchronous manner
var taskList = new List <Task <IRestResponse> >();
for (int i = 1; i < numPages; i++)
{
var ithPageUrl = Url + (Url.Contains("?") ? "&" : "?") + $"pagesize={pagesize}&$skip={i*pagesize}";
taskList.Add(GetAsync(ithPageUrl, ApiParameters));
LoggingFunction?.Invoke($"Initialized fetch of page {i + 1}/{numPages}");
}
// await the page tasks
for (int i = 1; i < numPages; i++)
{
// await the response and remove it from the list
var ithPageResponseIndex = Task.WaitAny(taskList.ToArray());
var ithPageResponseTask = taskList[ithPageResponseIndex];
taskList.Remove(ithPageResponseTask);
var ithPageResponse = await ithPageResponseTask;
if (!ithPageResponse.IsSuccessful)
{
throw new Exception($"Error retrieving page from API: {ithPageResponse.Content}");
}
// if successful, we can add the data to the feedData list
var ithPageResponseJson = (JObject)JsonConvert.DeserializeObject(ithPageResponse.Content);
((JArray)ithPageResponseJson["value"]).ToList().ForEach(j => feedData.Add((JObject)j));
LoggingFunction?.Invoke($"Successfully retrieved page {i + 1}/{numPages}");
}
return(feedData);
}
/// <summary>
/// Waits for a job to finish, returns the publish link when it does.
/// </summary>
/// <param name="BaseUrl">This should be either https://apieast.aims360.rest or https://apiwest.aims360.rest</param>
/// <param name="JobID">The job to rerun and get the publish link for</param>
/// <param name="ApiParameters">Changes the behavior of the Api calls</param>
/// <returns>The publish link. We need to return this in case it's a public link</returns>
public async Task <string> WaitForPublishLink(string BaseUrl, string JobID, ApiParametersContainer ApiParameters = null)
{
var JobNameString = ApiParameters?.JobName != null ? ApiParameters?.JobName + ": " : "";
// loop until the job is complete
string status = null;
string publishLink = null;
int i = 1;
do
{
// choose a random time between 3 and 7 seconds to sleep for to avoid overlap in parallel calls
int seed = Convert.ToInt32(DateTimeOffset.Now.ToUnixTimeMilliseconds() % Int32.MaxValue);
var randms = new Random(seed).Next(3000, 7000);
Thread.Sleep(randms);
// make the api call for the status of the job
var statusResponse = await this.GetAsync($"{BaseUrl}/jobsmanagement/v1.0/backgroundjob/{JobID}", ApiParameters);
if (!statusResponse.IsSuccessful)
{
throw new Exception($"Error fetching job status for {JobID}: {statusResponse.Content}");
}
// tell the user which iteration we're on
String iterationString = ApiParameters?.MaxNumIterations == null ? $"{i}" : $"{i}/{(ApiParameters?.MaxNumIterations ?? -1)}";
// get the status of the job
var responseJson = (JObject)JsonConvert.DeserializeObject(statusResponse.Content);
status = (string)responseJson["jobStatus"];
if (status != "Completed" && status != "Failed")
{
status = (string)responseJson["jobStatusText"];
}
publishLink = (string)responseJson["publishLink"];
LoggingFunction?.Invoke(JobNameString + $"Iteration {iterationString}. Status: {status}.");
i++;
// finish when the job is complete or if we've tried enough times (infinite if MaxNumIterations=null)
}while((status != "Completed" && status != "Failed") || (ApiParameters?.MaxNumIterations != null && i > (ApiParameters?.MaxNumIterations ?? -1)));
// edge case when we can't wait this long
if (status != "Completed")
{
throw new Exception($"{JobNameString}Job {JobID} was not successfully completed");
}
return(publishLink);
}
/// <summary>
/// Reruns the job assocated with the jobID parameter.
/// </summary>
/// <param name="BaseUrl">This should be either https://apieast.aims360.rest or https://apiwest.aims360.rest</param>
/// <param name="JobID">The job to rerun and get the publish link for</param>
/// <param name="ApiParameters">Changes the behavior of the Api calls</param>
public async Task RerunJob(string BaseUrl, string JobID, ApiParametersContainer ApiParameters = null)
{
var JobNameString = ApiParameters?.JobName != null ? ApiParameters?.JobName + ": " : "";
// rerun the job
LoggingFunction?.Invoke(JobNameString + "Rerunning job");
var rerunResponse = await this.PostAsync($"{BaseUrl}/jobsmanagement/v1.0/backgroundjob/{JobID}/rerun");
if (!rerunResponse.IsSuccessful)
{
throw new Exception($"Error rerunning aqua job {JobID}: {rerunResponse.Content}");
}
LoggingFunction?.Invoke(JobNameString + "Successfully reran job");
}
/// <summary>
/// Uses Newtonsoft.Json's SelectToken function to find a token in path.
/// This token path will be absolute, so make sure you construct your url
/// in such a way that it unambiguously returns 1 output. Remember that this
/// uses caching.
/// </summary>
/// <param name="Url">The exact url to call from the API</param>
/// <param name="JTokenPath">The exact static path to find the JToken at</param>
/// <param name="ApiParameters">Changes the behavior of the Api calls</param>
/// <returns>The JToken at the path, or null if not present</returns>
public async Task <JToken> GetTokenAsync(String Url, String JTokenPath, ApiParametersContainer ApiParameters = null)
{
// get the response from the API
var response = await this.GetAsync(Url, ApiParameters);
if (!response.IsSuccessful)
{
throw new Exception($"Could not retrieve data from {Url}: {response.Content}");
}
// parse the json and return the token
var responseJson = (JObject)JsonConvert.DeserializeObject(response.Content);
var token = responseJson.SelectToken(JTokenPath);
return(token);
}
/// <summary>
/// Calls the API generically using a semaphore and
/// has some basic caching. If the url matches a
/// previous call EXACTLY, it bypasses the API call
/// and simply returns the previous result.
/// </summary>
/// <param name="url">The complete url to retrieve the data from</param>
/// <param name="ApiParameters">Changes the behavior of the Api calls</param>
public async Task <IRestResponse> GetAsync(string url, ApiParametersContainer ApiParameters = null)
{
// check to see if we've used this call before
if ((ApiParameters?.UseCaching ?? false) && ApiCache.ContainsKey(url))
{
return(ApiCache[url]);
}
// make API call using the generic api call, which uses semaphores
var restClient = new RestClient(url);
var restRequest = new RestRequest(Method.GET);
restRequest.AddHeader("Authorization", Bearer);
var restResponse = await ExecuteAsync(restClient, restRequest, ApiParameters);
lock (ApiCache){
ApiCache[url] = restResponse;
}
return(restResponse);
}
/// <summary>
/// Gets data from the publish link (attaches a bearer token in case it's private) and
/// converts the resulting CSV into a data table. If no parameters are included,
/// </summary>
/// <param name="PublishLink">The publish link from which we will get the data</param>
/// <param name="ApiParameters">Changes the behavior of the Api calls</param>
/// <returns>A data table representing the data from the publish link</returns>
public async Task <DataTable> GetAquaCsvData(string PublishLink, ApiParametersContainer ApiParameters = null)
{
LoggingFunction?.Invoke($"Fetching CSV data from {PublishLink}");
// publish links should never be cached responses unless specified
var publishLinkResponse = await this.GetAsync(PublishLink, (ApiParameters ?? new ApiParametersContainer()
{
UseCaching = false
}));
if (!publishLinkResponse.IsSuccessful)
{
throw new Exception("Retrieval of data from publish link was unsuccessful: " + PublishLink);
}
// convert to CSV
var jobName = ApiParameters?.JobName == null ? "" : $"{ApiParameters?.JobName}: ";
LoggingFunction?.Invoke($"{jobName}Converting CSV data to DataTable");
var tableFromCsv = DataTableUtility.ParseFromCsv(publishLinkResponse.Content);
LoggingFunction?.Invoke($"{jobName}Successfully converted CSV data to DataTable");
return(tableFromCsv);
}
/// <summary>
/// Reruns the aqua job and gets the job publish link after waiting for completion.
/// </summary>
/// <param name="BaseUrl">This should be either https://apieast.aims360.rest or https://apiwest.aims360.rest</param>
/// <param name="JobID">The job to rerun and get the publish link for</param>
/// <param name="ApiParameters">Changes the behavior of the Api calls</param>
/// <returns>The publish link. We need to return this in case it's a public link</returns>
public async Task <string> RerunAndWaitForJobPublishLink(string BaseUrl, string JobID, ApiParametersContainer ApiParameters = null)
{
await RerunJob(BaseUrl, JobID, ApiParameters);
var publishLink = await WaitForPublishLink(BaseUrl, JobID, ApiParameters);
return(publishLink);
}
/// <summary>
/// The most generic api call you could ever make:
/// you provide the client and request, this just handles retries.
/// </summary>
/// <param name="restClient"></param>
/// <param name="restRequest"></param>
/// <param name="ApiParameters">Changes the behavior of the Api calls</param>
/// <returns></returns>
public async Task <IRestResponse> ExecuteAsync(RestClient restClient, RestRequest restRequest, ApiParametersContainer ApiParameters = null)
{
IRestResponse restResponse = null;
// main loop for retries
for (int i = 0; i < (ApiParameters?.MaxNumIterations ?? 3); i++)
{
_semaphore?.WaitOne(); // allow up to 5 threads to call aims at a time, so as to not overload the servers
restResponse = await restClient.ExecuteAsync(restRequest);
_semaphore?.Release();
if (restResponse.IsSuccessful)
{
return(restResponse);
}
// exponential backoff
else
{
var timeInSeconds = (int)Math.Pow(2, i);
Thread.Sleep(timeInSeconds * 1000);
}
// make this an infinite infinite loop if maxnumiterations is set to null
if (ApiParameters != null && ApiParameters.MaxNumIterations == null)
{
i--;
}
}
return(restResponse);
}
/// <summary>
/// Calls the API generically using a semaphore. Adds the bearer
/// token to the call.
/// </summary>
/// <param name="Url">The complete url to retrieve the data from</param>
/// <param name="JsonPayload">The body (json format) to send with the POST request</param>
/// <param name="ApiParameters">Changes the behavior of the Api calls</param>
public async Task <IRestResponse> PostAsync(string Url, string JsonPayload = null, ApiParametersContainer ApiParameters = null)
{
var restClient = new RestClient(Url);
var restRequest = new RestRequest(Method.POST);
if (JsonPayload != null)
{
restRequest.AddJsonBody(JsonPayload);
}
restRequest.AddHeader("Authorization", Bearer);
return(await this.ExecuteAsync(restClient, restRequest, ApiParameters));
}
