public bool StopTensorboard(string id)
{
bool result = false;
string ResourceName = $"{id}.pmml";
string ResourceNameWithoutExtension = id;
string ResourcePath = DirectoryHelper.GetModelDirectoryPath() + ResourceName;
string Message = "Error";
System.Console.WriteLine("Resource Path " + ResourcePath);
Dictionary <string, string> Result = new Dictionary <string, string>();
try
{
TensorBoard TBTool = this.tbClient.GetTensorBoardTool();
ITask TBNoteBookTask = TBTool.FindTask(ResourcePath);
if (TBNoteBookTask.IsEmpty())
{
Message = "Error : There is no such task running";
}
else
{
TBTool.StopTask(ResourcePath);
result = true;
InstancePayload.Delete(id);
Message = "Tensorboard successfully stop";
}
}
catch (Exception ex)
{
Message = ex.Message;
}
return(result);
}
public IActionResult DeleteInstancesAsync(string id)
{
bool result = false;
string type = "";
//check file type to delete - jupyter
var codeResponse = CodePayload.Get();
foreach (var item in codeResponse)
{
if (item.Id == id)
{
type = item.Type;
}
}
//check got pmml file
if (string.IsNullOrEmpty(type))
{
var modalResponse = ModelPayload.Get();
foreach (var item in modalResponse)
{
if (item.Id == id)
{
type = item.Type;
}
}
}
//check for zmk
if (string.IsNullOrEmpty(type))
{
type = "ZMK";
}
switch (type)
{
case "JUPYTER_NOTEBOOK":
result = StopJupyter(id);
InstancePayload.Delete(id);
break;
case "PMML":
result = StopTensorboard(id);
InstancePayload.Delete(id);
break;
case "ZMK":
result = ZMKDockerCmdHelper.StopZMKInstance(id);
break;
}
var zmkResponse1 = InstancePayload.Get();
return(Ok(new { user = string.Empty, id = id, type = type, message = "Instance deleted successfully.", Json = JsonConvert.SerializeObject(zmkResponse1) }));
}
public bool StopJupyter(string id)
{
bool result = false;
string ResourceName = $"{id}.ipynb";
string ResourceNameWithoutExtension = id;
string notebookDir = DirectoryHelper.GetCodeDirectoryPath() + ResourceNameWithoutExtension;
string ResourcePath = notebookDir + System.IO.Path.DirectorySeparatorChar + ResourceName;
string Message = "Error";
System.Console.WriteLine("Resource Path " + ResourcePath);
Dictionary <string, string> Result = new Dictionary <string, string>();
//
var obj = new
{
base_url = "/",
NotebookDir = $"{notebookDir}",
ResourcePath = $"{ResourcePath}"
};
//
try
{
JupyterNotebook JNBTool = this.jupyterClient.GetJupyterNotebookTool();
ITask JupyterNoteBookTask = JNBTool.FindTask(ResourcePath);
if (JupyterNoteBookTask.IsEmpty())
{
Message = "Error : There is no such task running";
// var inst = InstancePayload.Get();
// foreach(var item in inst)
// {
// if(item.Id == id)
// {
// InstancePayload.Delete(id);
// result = true;
// }
// }
}
else
{
JNBTool.StopTask(ResourcePath);
result = true;
InstancePayload.Delete(id);
Message = "Notebook successfully stop";
}
}
catch (Exception ex)
{
Message = ex.Message;
}
// Result.Add("Result", Message);
return(result);
}
public async Task <IActionResult> GetJupyterNotebookUrlAsync(string id)
{
string response = string.Empty;
string _jUrl = string.Empty;
string message = "Notebook is up and running successfully";
//We needs to populate Current User from request
string CURRENT_USER = string.Empty;
string ResourcePath = Helpers.Common.FilePathHelper.GetFilePathById(id, codeResponse);
FileInfo resourceInfo = new System.IO.FileInfo(ResourcePath);
string notebookDir = resourceInfo.Directory.ToString();
string notebookLinkURL = string.Empty;
var obj = new
{
base_url = "/",
NotebookDir = $"{notebookDir}",
ResourcePath = $"{ResourcePath}"
};
try
{
var portRegex = new Regex(@"(?<![^/]/[^/]*):\d+");//to remove port number
JupyterNotebook JNBTool = this.jupyterClient.GetJupyterNotebookTool();
await System.Threading.Tasks.Task.FromResult(0);
ITask JupyterNoteBookTask = JNBTool.FindTask(ResourcePath);
if (JupyterNoteBookTask.IsEmpty())
{
JNBTool.StartTaskAsync((int)TaskTypes.Start, ResourcePath, (JObject)JObject.FromObject(obj));
}
notebookLinkURL = JNBTool.GetResourceLink(ResourcePath);
// notebookLinkURL = @"C:\myCode\Project\ZMOD\code\1_SVM\1_SVM.ipynb";
//
// JupyterNotebook nb = JupyterNotebook.Find(ResourcePath);
var objJNBInst = new InstanceResponse()
{
Id = id,
Name = $"{id}.ipynb",
Type = "JNB"
// Properties = new List<InstanceProperty>(){ new InstanceProperty{ key="port", value=nb.Info.Port}}
};
InstancePayload.Create(objJNBInst);
notebookLinkURL = notebookLinkURL.Replace(@"//", @"/");
//
return(Ok(new { user = CURRENT_USER, id = id, message = message, url = notebookLinkURL.Replace(notebookLinkURL.Substring(0, notebookLinkURL.IndexOf("jnb")), "") }));
}
catch (Exception ex)
{
message = ex.Message;
return(BadRequest(new { user = CURRENT_USER, id = id, message = message }));
}
}
public async Task <IActionResult> StartInstancesAsync()
{
string reqBody = "";
string instanceType = "";
// string cmd ="";
try
{
//read request body
using (var reader = new StreamReader(Request.Body))
{
var body = await reader.ReadToEndAsync();
reqBody = body.ToString();
}
//
IList <InstanceResponse> getAllInstances = InstancePayload.Get();
//
JObject jObj = JObject.Parse(reqBody);
instanceType = jObj["type"].ToString();
switch (instanceType)
{
case "ZMK":
#region start ZMK instances
try
{
string unassigned = ZMKDockerCmdHelper.GetUnassignedZMKInstance();
if (!string.IsNullOrEmpty(unassigned))
{
ZMKDockerCmdHelper.StartZMKInstance(unassigned);
}
}
catch (Exception ex)
{
return(BadRequest(new { message = "starting instance failed.", exception = ex.StackTrace }));
}
#endregion
break;
}
}
catch (Exception ex)
{
return(BadRequest(new { message = "starting instance failed.", exception = ex.StackTrace }));
}
return(Ok());
}
public IActionResult Delete(string id)
{
if (InstancePayload.IsInstanceExists(id) && CodePayload.GetById(id).Type == "JUPYTER_NOTEBOOK")
{
return(BadRequest(new { user = string.Empty, id = id, message = "Error deleting file. Instance of the file is running. Check Assests module" }));
}
bool result = CodePayload.Delete(id);
if (result == true)
{
return(Ok(new { user = string.Empty, id = id, message = "File deleted successfully." }));
}
else
{
return(BadRequest(new { user = string.Empty, id = id, message = "Error deleting file. Try again or contact adminstrator." }));
}
}
public IActionResult GetRunningInstances()
{
string cmd = "";
string output = "";
List <InstanceResponse> getAllInstances = InstancePayload.Get();
#region GPU listing
cmd = "gpustat --json";
try
{
output = cmd.Bash();
if (!string.IsNullOrEmpty(output))
{
JObject jsonObj = JObject.Parse(output);
JArray arr = (JArray)jsonObj["gpus"];
int gpuctr = 1;
foreach (var a in arr)
{
JArray tmpArr = (JArray)a["processes"];
List <InstanceProperty> _props = new List <InstanceProperty>();
_props.Add(new InstanceProperty {
key = "temperature.gpu [celsius]", value = a["temperature.gpu"]
});
_props.Add(new InstanceProperty {
key = "utilization.gpu [%]", value = a["utilization.gpu"]
});
_props.Add(new InstanceProperty {
key = "power.draw [W]", value = a["power.draw"]
});
_props.Add(new InstanceProperty {
key = "enforced.power.limit [W]", value = a["enforced.power.limit"]
});
_props.Add(new InstanceProperty {
key = "memory.used [MB]", value = a["memory.used"]
});
_props.Add(new InstanceProperty {
key = "memory.total [MB]", value = a["memory.total"]
});
InstanceResponse _instance = new InstanceResponse()
{
Id = a["uuid"].ToString(),
Name = $"GPU {gpuctr}",
Type = "GPU",
Properties = _props,
Processes = tmpArr
};
getAllInstances.Add(_instance);
gpuctr++;
}
}
}
catch (Exception ex)
{
return(BadRequest(new { message = "running instance loading failed.", exception = ex.StackTrace }));
}
#endregion
#region ZMK listing
IList <InstanceResponse> zmkList = ZMKDockerCmdHelper.GetAllRunningZMK();
if (zmkList.Count > 0)
{
getAllInstances.AddRange(zmkList);
}
#endregion
return(Json(getAllInstances));
}
public async Task <IActionResult> PostModelTrainAsync(string id)
{
string response = string.Empty;
string dirFullpath = DirectoryHelper.GetDataDirectoryPath();//create new folder in data folder with the name of the file
string reqBody = string.Empty;
string filePath = string.Empty;
string dataFolder = string.Empty;
JObject jObjOrig = new JObject();
using (var reader = new StreamReader(Request.Body))
{
var body = reader.ReadToEnd();
reqBody = body.ToString();
}
try
{
//get file name
if (responseData.Count > 0)
{
foreach (var record in responseData)
{
if (record.Id.ToString() == id)
{
//set the filePath
filePath = record.FilePath;
//create dir with filename in data folder
if (!Directory.Exists(dirFullpath + record.Name.Replace(".pmml", string.Empty)))
{
Directory.CreateDirectory(dirFullpath + record.Name.Replace(".pmml", string.Empty));
dataFolder = dirFullpath + record.Name.Replace(".pmml", string.Empty);
}
}
}
}
//json merge
//Add TensorBoard Info
if (!string.IsNullOrEmpty(reqBody))
{
jObjOrig = JObject.Parse(reqBody);
}
if (!string.IsNullOrEmpty(filePath))
{
string ResourcePath = filePath;
string TensorBoardLink = string.Empty;
string TensorboardLogFolder = string.Empty;
var obj = new
{
base_url = "/",
ResourcePath = $"{ResourcePath}"
};
try
{
var portRegex = new Regex(@"(?<![^/]/[^/]*):\d+");//to remove port number
TensorBoard TBTool = this.tbClient.GetTensorBoardTool();
ZMM.Tasks.ITask TensorBoardTask = TBTool.FindTask(ResourcePath);
if (TensorBoardTask.IsEmpty())
{
TBTool.StartTaskAsync((int)TaskTypes.Start, ResourcePath, (JObject)JObject.FromObject(obj));
}
TensorBoardLink = TBTool.GetResourceLink(ResourcePath, out TensorboardLogFolder);
//
//TB redirection
string tbLink = "";
if (TensorBoardLink.Contains("6006"))
{
tbLink = TensorBoardLink.Replace(":6006", "/tb1");
}
else if (TensorBoardLink.Contains("6007"))
{
tbLink = TensorBoardLink.Replace(":6007", "/tb2");
}
else if (TensorBoardLink.Contains("6008"))
{
tbLink = TensorBoardLink.Replace(":6008", "/tb3");
}
//
jObjOrig.Add("filePath", ResourcePath);
jObjOrig.Add("tensorboardLogFolder", TensorboardLogFolder);
jObjOrig.Add("tensorboardUrl", tbLink);
Logger.LogInformation("PostModelTrainAsync", jObjOrig.ToString());
//for asset
// int sIdx = TensorBoardLink.IndexOf(":6");
// var tbInst = new List<InstanceProperty>();
// tbInst.Add(new InstanceProperty(){ key = "port", value = TensorBoardLink.Substring(sIdx,6)});
//
var objJNBInst = new InstanceResponse()
{
Id = id,
Name = $"{id}",
Type = "TB"
};
InstancePayload.Create(objJNBInst);
//
}
catch (Exception ex)
{
Logger.LogCritical("PostModelTrainAsync", ex.Message);
return(BadRequest(new { user = CURRENT_USER, id = id, message = ex.Message }));
}
}
if (!string.IsNullOrEmpty(dataFolder))
{
jObjOrig.Add("dataFolder", dataFolder);
}
response = await nnclient.TrainModel(jObjOrig.ToString());
return(Json(response));
}
catch (Exception ex)
{
//TO DO: ILogger
string _ex = ex.Message;
return(BadRequest());
}
}
