public async Task <string> GetModels(string zmodId)
{
string jsonResult = string.Empty;
var tuple = ZSSettingPayload.GetUserInfo(zmodId);
var auth = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{tuple.Item2}:{tuple.Item3}")));
using (var httpClient = new HttpClient())
{
httpClient.BaseAddress = new System.Uri(tuple.Item1);
httpClient.DefaultRequestHeaders.Accept.Clear();
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
httpClient.DefaultRequestHeaders.Authorization = auth;
try
{
HttpResponseMessage response = await httpClient.GetAsync("service/zementis/models");
if (response.IsSuccessStatusCode)
{
jsonResult = await response.Content.ReadAsStringAsync();
}
else
{
return(ZMMConstants.ErrorFailed);
}
}
catch (HttpRequestException ex)
{
jsonResult = "{'message': '" + ex.Message + "', 'error':'" + ZMMConstants.ErrorFailed + "'}";
}
}
return(jsonResult);
}
public async Task <string> UploadPmml(string filePath, string zmodId)
{
string jsonResult = string.Empty;
var tuple = ZSSettingPayload.GetUserInfo(zmodId);
var auth = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{tuple.Item2}:{tuple.Item3}")));
try
{
using (var httpClient = new HttpClient())
{
httpClient.BaseAddress = new System.Uri(tuple.Item1);
httpClient.DefaultRequestHeaders.Accept.Clear();
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
httpClient.DefaultRequestHeaders.Authorization = auth;
httpClient.Timeout = TimeSpan.FromMinutes(30);
using (var content = new MultipartFormDataContent("Upload--" + DateTime.Now.ToString(CultureInfo.InvariantCulture)))
{
var memory = new MemoryStream();
using (var stream = new FileStream(filePath, FileMode.Open))
{
await stream.CopyToAsync(memory);
}
memory.Position = 0;
content.Add(new StreamContent(memory), "file", filePath);
HttpResponseMessage response = await httpClient.PostAsync("service/zementis/model", content);
if (response.IsSuccessStatusCode)
{
jsonResult = await response.Content.ReadAsStringAsync();
}
else if (response.StatusCode.ToString() == "402")
{
return("FileExists");
}
else
{
return("Fail");
}
}
}
}
catch (Exception ex)
{
string _ex = ex.Message;
return("fail");
}
return(jsonResult);
}
public async Task <IActionResult> AddSettingAsync()
{
string jsonBody = "", zmodId = "";
//read from body
using (var reader = new StreamReader(Request.Body))
{
var body = await reader.ReadToEndAsync();
jsonBody = body.ToString();
}
if (!string.IsNullOrEmpty(jsonBody))
{
if (XSSBlackList.CheckString(jsonBody))
{
return(BadRequest(new { message = "Error: Unable to save settings. Invalid input. Please enter correct settings.", errorCode = 400 }));
}
zmodId = ZSSettingPayload.GetUserNameOrEmail(HttpContext);
//parse
JObject jObj = JObject.Parse(jsonBody);
JArray jArr = (JArray)jObj["settings"];
var setList = jArr.ToObject <List <SettingProperty> >();
//fetch the original record
List <SettingProperty> setListOrig = ZSSettingPayload.GetSettingsByUser(zmodId).SelectMany(b => b.Settings).ToList <SettingProperty>();
foreach (var p in setList)
{
if (p.username.Contains("******"))
{
p.username = setListOrig.Where(c => c.url == p.url).Select(c => c.username).FirstOrDefault().ToString();
}
if (p.password.Contains("******"))
{
p.password = setListOrig.Where(c => c.url == p.url).Select(c => c.password).FirstOrDefault().ToString();
}
}
//add to payload
var newRecord = new ZSSettingResponse()
{
ZmodId = zmodId,
Settings = setList
};
ZSSettingPayload.CreateOrUpdate(newRecord);
return(Json(JObject.Parse(jsonBody)));
}
else
{
return(NotFound());
}
}
public async Task <string> DeployModelAsync(string zmodId, string filePath)
{
string jsonResult = string.Empty;
var tuple = ZSSettingPayload.GetUserCredetials(zmodId, "MLE-ONNX");
var auth = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{tuple.Item2}:{tuple.Item3}")));
using (var httpClient = new HttpClient())
{
httpClient.BaseAddress = new System.Uri(tuple.Item1);
httpClient.DefaultRequestHeaders.Accept.Clear();
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
httpClient.DefaultRequestHeaders.Authorization = auth;
httpClient.Timeout = TimeSpan.FromMinutes(30);
using (var content = new MultipartFormDataContent("Upload--" + DateTime.Now.ToString(CultureInfo.InvariantCulture)))
{
var memory = new MemoryStream();
using (var stream = new FileStream(filePath, FileMode.Open))
{
await stream.CopyToAsync(memory);
}
memory.Position = 0;
content.Add(new StreamContent(memory), "file", filePath);
HttpResponseMessage response = await httpClient.PostAsync("service/zementis/onnx/models", content);
if (response.IsSuccessStatusCode)
{
jsonResult = await response.Content.ReadAsStringAsync();
}
else if (response.StatusCode.ToString() == "Conflict")
{
return($"Fail@@A model with the same name already exists.");
}
else
{
return($"Fail@@Something went wrong while deploying to MLE. Please try again.");
}
}
}
return(jsonResult);
}
public async Task <string> ImageScoring(string modelName, string filePath, string zmodId)
{
string jsonResult = string.Empty;
var tuple = ZSSettingPayload.GetUserInfo(zmodId);
var auth = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{tuple.Item2}:{tuple.Item3}")));
using (var httpClient = new HttpClient())
{
httpClient.BaseAddress = new System.Uri(tuple.Item1);
httpClient.DefaultRequestHeaders.Accept.Clear();
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
httpClient.DefaultRequestHeaders.Authorization = auth;
// httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("*****@*****.**", "softwareag");
//
try
{
using (var content = new MultipartFormDataContent())
{
var memory = new MemoryStream();
using (var stream = new FileStream(filePath, FileMode.Open))
{
await stream.CopyToAsync(memory);
}
memory.Position = 0;
content.Add(new StreamContent(memory), "file", filePath);
HttpResponseMessage response = await httpClient.PostAsync($"service/zementis/apply/{modelName}", content);
if (response.IsSuccessStatusCode)
{
jsonResult = await response.Content.ReadAsStringAsync();
}
}
}
catch (HttpRequestException ex)
{
//add log
jsonResult = "{'message': '" + ex.Message + "'}";
}
}
return(jsonResult);
}
public async Task <string> DeletePmml(string modelName, string zmodId)
{
string jsonResult = string.Empty;
var tuple = ZSSettingPayload.GetUserInfo(zmodId);
var auth = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{tuple.Item2}:{tuple.Item3}")));
using (var httpClient = new HttpClient())
{
httpClient.BaseAddress = new System.Uri(tuple.Item1);
httpClient.DefaultRequestHeaders.Accept.Clear();
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
httpClient.DefaultRequestHeaders.Authorization = auth;
//
var content = new FormUrlEncodedContent(
new List <KeyValuePair <string, string> >
{
//nothing
}
);
//
try
{
HttpResponseMessage response = await httpClient.DeleteAsync("service/zementis/model/" + modelName);
if (response.IsSuccessStatusCode)
{
jsonResult = await response.Content.ReadAsStringAsync();
}
}
catch (HttpRequestException ex)
{
jsonResult = "{'message': '" + ex.Message + "'}";
}
}
return(jsonResult);
}
public async Task <IActionResult> GetSettingsAsync(string type, bool selected, bool unmask, bool showall)
{
//get the zmodId
string UserEmailId = ZSSettingPayload.GetUserNameOrEmail(HttpContext);
// string UserEmailId = "*****@*****.**";
JObject jObj = new JObject();
await Task.FromResult(0);
var settings = ZSSettingPayload.GetSettingsByUser(UserEmailId);
List <SettingProperty> settingProperties;
if (string.IsNullOrEmpty(type))
{
settingProperties = settings.SelectMany(b => b.Settings).ToList <SettingProperty>();
}
else
{
settingProperties = settings.SelectMany(b => b.Settings).ToList <SettingProperty>();
if (!string.IsNullOrEmpty(type) && (showall == true))
{
settingProperties = settingProperties.Where(c => c.type == $"{type}").ToList <SettingProperty>();
}
else
{
settingProperties = settingProperties.Where(c => c.type == $"{type}" && c.selected == selected).ToList <SettingProperty>();
}
}
// var selectedType = settingProperties.Where(c=>c.type == $"{type}").ToList<SettingProperty>();
//
#region seed settings
if (settings.Count == 0)
{
var template = new ZSSettingResponse
{
ZmodId = UserEmailId,
Settings = new List <SettingProperty> {
new SettingProperty {
name = "Cumulocity", type = "C8Y", tenantID = "ai", username = "******", password = "******", url = "https://ai.eu-latest.cumulocity.com", selected = true
},
new SettingProperty {
name = "Cumulocity", type = "C8Y", tenantID = "ai", username = "******", password = "******", url = "https://ai.cumulocity.com", selected = false
},
new SettingProperty {
name = "Predictive Analytics-PMML", type = "MLE-PMML", tenantID = "zserver", username = "******", password = "******", url = "https://ai.eu-latest.cumulocity.com/", selected = true
},
new SettingProperty {
name = "Repo Server 1", type = "NR", tenantID = "repo", username = "******", password = "******", url = "https://repo.umoya.ai/", selected = false
},
new SettingProperty {
name = "Repo Server 2", type = "NR", tenantID = "hub", username = "******", password = "******", url = "https://hub.umoya.ai/", selected = true
},
new SettingProperty {
name = "DataHub 1", type = "DH", driver = "Dremio", username = "******", password = "******", url = "https://url", port = "0000", selected = true, ssl = "1"
},
new SettingProperty {
name = "Predictive Analytics-ONNX", type = "MLE-ONNX", username = "******", password = "******", url = "https://your-tenent-id/", selected = true
}
}
};
jObj = JObject.Parse(JsonConvert.SerializeObject(template));
ZSSettingPayload.CreateOrUpdate(template);
}
else
{
var template = new ZSSettingResponse
{
ZmodId = UserEmailId,
Settings = settingProperties
};
jObj = JObject.Parse(JsonConvert.SerializeObject(template));
}
#endregion
jObj.Remove("zmodId");
//
foreach (var p in jObj["settings"])
{
if (unmask == false)
{
p["username"] = "******";
p["password"] = "******";
}
if (p["type"].ToString() == "DH")
{
p["tenantID"].Parent.Remove();
}
else
{
p["port"].Parent.Remove();
p["driver"].Parent.Remove();
}
if (p["ssl"].ToString() == "")
{
p["ssl"].Parent.Remove();
}
}
//
return(Json(jObj));
}
public async Task <IActionResult> PostSqlAsync()
{
DataResponse newRecord = new DataResponse();
string reqBody = "";
string reqSql = "";
string dirFullpath = $"{DirectoryHelper.GetDataDirectoryPath()}";
string newFile = "DataHub_" + DateTime.Now.Ticks.ToString() + ".csv";
string _filePath = Path.Combine(dirFullpath, newFile);
long fileSize = 0L;
List <string> resultRows = new List <string>();
StringBuilder csvBuilder = new StringBuilder();
List <Property> _props = new List <Property>();
int numberOfColumns = 0;
//
using (var reader = new StreamReader(Request.Body))
{
reqBody = reader.ReadToEnd().ToString();
}
JObject jsonBody = JObject.Parse(reqBody);
reqSql = jsonBody["sql"].ToString();
#region ODBC
string zmodId = ZSSettingPayload.GetUserNameOrEmail(HttpContext);
var settings = ZSSettingPayload.GetDataHubInfo(zmodId);
string cnn = $"Driver={settings.Item5};ConnectionType=Direct;HOST={settings.Item1};PORT={settings.Item4};AuthenticationType=Plain;UID={settings.Item2};PWD={settings.Item3};SSL={settings.Item6}";
Console.WriteLine($"DATAHUB >>>>>>>>>>>>>>>>>>>>> { cnn }");
//using (OdbcConnection connection = new OdbcConnection("Driver=Dremio Connector;ConnectionType=Direct;HOST=dremio-demo.westeurope.cloudapp.azure.com;PORT=31010;AuthenticationType=Plain;UID=demo;PWD=iug2019#riga"))
using (OdbcConnection connection = new OdbcConnection(cnn))
{
try
{
connection.Open();
Console.WriteLine("DATAHUB CONNECTION ESTABLISHED...");
OdbcCommand DbCommand = connection.CreateCommand();
Console.WriteLine(reqSql);
DbCommand.CommandText = reqSql;
List <string> columns = new List <string>();
OdbcDataReader reader = DbCommand.ExecuteReader();
numberOfColumns = reader.FieldCount;
//add header column name
for (int i = 0; i < reader.FieldCount; i++)
{
columns.Add(reader.GetName(i));
csvBuilder.Append(reader.GetName(i));
if (i < reader.FieldCount - 1)
{
csvBuilder.Append(",");
}
}
resultRows.Add(csvBuilder.ToString());
csvBuilder.Clear();
//
//add rows
while (reader.Read())
{
csvBuilder.Clear();
for (int i = 0; i < reader.FieldCount; i++)
{
if (!reader.IsDBNull(i))
{
csvBuilder.Append(reader[i]);
}
else
{
csvBuilder.Append("No Data");
}
if (i < reader.FieldCount - 1)
{
csvBuilder.Append(",");
}
}
resultRows.Add(csvBuilder.ToString());
}
using (StreamWriter writer = new StreamWriter(_filePath))
{
foreach (var line in resultRows)
{
writer.WriteLine(line);
}
writer.Flush();
fileSize = writer.BaseStream.Length;
}
string _url = DirectoryHelper.GetDataUrl(newFile);
await Task.FromResult(0);
//
string type = "CSV";
//get properties row and column count
int[] csvProps = CsvHelper.GetCsvRowColumnCount(dirFullpath + @"/" + newFile);
_props.Add(new Property {
key = "Number of rows", value = resultRows.Count().ToString()
});
_props.Add(new Property {
key = "Number of columns", value = numberOfColumns.ToString()
});
newRecord = new DataResponse()
{
Created_on = DateTime.Now.ToString(),
Edited_on = DateTime.Now.ToString(),
Extension = "CSV",
FilePath = _filePath,
Id = newFile,
MimeType = "text/csv",
Name = newFile.Replace(".CSV", ""),
Properties = _props,
Size = fileSize,
Type = type,
Url = _url,
DateCreated = DateTime.Now
};
DataPayload.Create(newRecord);
}
catch (Exception e)
{
// return error message
Console.WriteLine("DataHub ERROR:>>>>" + e.Message);
//cleanup the file created
}
}
#endregion
//
return(Json(newRecord));
}