public static IONAPI LoadIONAPI(string connection)
{
string connInfo = System.IO.File.ReadAllText(connection);
dynamic dyn = JsonConvert.DeserializeObject(connInfo);
if (dyn.saak == null || dyn.sask == null)
{
return(null);
}
string or = dyn.or;
string cn = dyn.cn;
string ci = dyn.ci;
string cs = dyn.cs;
string pu = dyn.pu;
string saak = dyn.saak;
string sask = dyn.sask;
string oa = dyn.oa;
string ot = dyn.ot;
IONAPI quickAPI = new IONAPI(cn, ci, cs, pu, saak, sask, oa, ot, or);
return(quickAPI);
}
private static void LoadIonapi(string connection)
{
Console.WriteLine("Loading IONAPI configuration...");
string connInfo = System.IO.File.ReadAllText(connection);
dynamic dyn = JsonConvert.DeserializeObject(connInfo);
string or = dyn.or;
string application = dyn.cn;
string ci = dyn.ci;
string cs = dyn.cs;
string pu = dyn.pu;
string saak = dyn.saak;
string sask = dyn.sask;
string oa = dyn.oa;
string ot = dyn.ot;
_ionApi = new IONAPI(application, ci, cs, pu, saak, sask, oa, ot, or);
}
public static async Task <DataTable> RunDataLakeAsync(string sqlFile, IONAPI ionAPI)
{
string responseBody;
string Status;
dynamic columnTypes;
dynamic dynDataReturn;
string SQL = "";
Boolean.TryParse(Environment.GetEnvironmentVariable("DL_QuickDataTable"), out bool quickSave);
if (quickSave && File.Exists("DL_QuickDataTable.tmp"))
{
try
{
responseBody = File.ReadAllText("DL_QuickDataTable.tmp");
columnTypes = File.ReadAllText("DL_QuickDataColumns.tmp");
SQL = File.ReadAllText("DL_QuickSQL.tmp", Encoding.UTF8);
if (SQL.Equals(File.ReadAllText(sqlFile, Encoding.UTF8)))
{
dynDataReturn = JsonConvert.DeserializeObject(columnTypes);
columnTypes = dynDataReturn.columns;
Status = "Quick";
goto QuickDataLoad;
}
}
catch (Exception ex)
{ //Do nothing if lookup fails
Console.WriteLine("Warning: " + ex.Message);
}
}
string bearerToken = ionAPI.GetBearerAuthorization();
if (bearerToken == null)
{
Console.WriteLine("Error getting IONAPI Token: " + ionAPI.LastMessage);
Environment.Exit(-23);
}
Console.WriteLine("Loading Compass SQL File : " + sqlFile);
if (File.Exists(sqlFile))
{
SQL = File.ReadAllText(sqlFile, Encoding.UTF8);
}
else
{
Console.WriteLine("Compass Query file not found " + sqlFile);
Environment.Exit(-22);
}
if (quickSave)
{
File.WriteAllText("DL_QuickSQL.tmp", SQL, Encoding.UTF8);
}
Console.WriteLine("Calling Compass API...");
Task <HttpResponseMessage> query =
Program.CompassURL
.AppendPathSegment("/v1/compass/jobs")
.SetQueryParam("resultFormat", "application/json")
.WithOAuthBearerToken(bearerToken)
.WithHeader("User-Agent", "DataMover")
.WithHeader("Accept-Encoding", "deflate")
.WithHeader("Cache-Control", "no-cache")
.AllowAnyHttpStatus()
.SendStringAsync(HttpMethod.Post, SQL);
if (!query.Result.IsSuccessStatusCode)
{
Console.WriteLine("Compass API: " + query.Result.ReasonPhrase);
return(null);
}
responseBody = await query.ReceiveString();
dynDataReturn = JsonConvert.DeserializeObject(responseBody);
Status = dynDataReturn.status;
string Location = dynDataReturn.location;
string queryID = dynDataReturn.queryId;
Console.Write("Compass API: " + dynDataReturn.status);
query =
Program.CompassURL
.AppendPathSegment("/v1/compass/jobs/" + queryID + "/status", false)
.SetQueryParam("timeout", "1")
.WithOAuthBearerToken(bearerToken)
.WithHeader("User-Agent", "DataMover")
.WithHeader("Accept-Encoding", "deflate")
.WithHeader("Cache-Control", "no-cache")
.AllowAnyHttpStatus()
.GetAsync();
if (!query.Result.IsSuccessStatusCode)
{
Console.WriteLine("Compass API: " + query.Result.ReasonPhrase);
return(null);
}
responseBody = await query.ReceiveString();
dynDataReturn = JsonConvert.DeserializeObject(responseBody);
Status = dynDataReturn.status;
int loops = 0;
while (!Status.Equals("FINISHED") && !Status.Equals("FAILED"))
{
query =
Program.CompassURL
.AppendPathSegment("/v1/compass/jobs/" + queryID + "/status", false)
.SetQueryParam("timeout", "1")
.WithOAuthBearerToken(bearerToken)
.WithHeader("User-Agent", "DataMover")
.WithHeader("Accept-Encoding", "deflate")
.AllowAnyHttpStatus()
.GetAsync();
if (!query.Result.IsSuccessStatusCode)
{
Console.WriteLine("Compass API: " + query.Result.ReasonPhrase);
return(null);
}
responseBody = await query.ReceiveString();
dynDataReturn = JsonConvert.DeserializeObject(responseBody);
if (dynDataReturn.status != Status)
{
Console.WriteLine("");
Console.Write("Compass API: " + dynDataReturn.status);
}
Status = dynDataReturn.status;
System.Threading.Thread.Sleep(1000);
Console.Write(".");
loops++;
}
Console.WriteLine("");
columnTypes = dynDataReturn.columns;
if (quickSave)
{
File.WriteAllText("DL_QuickDataColumns.tmp", responseBody);
}
Console.WriteLine("Result ID : " + queryID + "");
query =
Program.CompassURL
.AppendPathSegment("/v1/compass/jobs/" + queryID + "/result", false)
.WithOAuthBearerToken(bearerToken)
.WithHeader("Connection", "keep-alive")
.WithHeader("Cache-Control", "no-cache")
.WithHeader("User-Agent", "DataMover")
.WithHeader("Accept-Encoding", "Deflate")
.WithHeader("Accept", "application/json")
.WithTimeout(600)
.AllowAnyHttpStatus()
.GetAsync();
if (!query.Result.IsSuccessStatusCode)
{
Console.WriteLine("Compass API: " + query.Result.ReasonPhrase);
return(null);
}
responseBody = await query.ReceiveString();
if (quickSave)
{
File.WriteAllText("DL_QuickDataTable.tmp", responseBody);
}
QuickDataLoad:
DataTable thisData = new DataTable("ExportMI");
if (Status.Equals("FAILED"))
{
Console.WriteLine("Compass API: " + responseBody);
return(null);
}
var jsonReader = new JsonTextReader(new StringReader(responseBody))
{
SupportMultipleContent = true // This is important!
};
thisData = JObject.Parse(responseBody)["data"].ToObject <DataTable>();
return(thisData);
}
