public void TestCreateModelpayload()
{
List <Property> _prop = new List <Property>();
List <ModelResponse> _model = new List <ModelResponse>();
_model = ModelPayload.Get();
ModelResponse newRecord = new ModelResponse()
{
Id = "HelloModel",
Name = "Hello.pmml",
User = "",
Created_on = DateTime.Now.ToString(),
Edited_on = DateTime.Now.ToString(),
Extension = ".png",
MimeType = "application/image",
Size = 111,
Type = "PMML",
Url = "http://localhost/uploads/data/Hello.png",
FilePath = "",
Loaded = false,
Deployed = false,
Properties = _prop
};
ModelResponse createdRecord = ModelPayload.Create(newRecord);
Assert.Equal(newRecord, createdRecord);
}
public static bool ScanModelsDirectory()
{
bool result = false;
string fileName, _url, fileExt = "";
Console.WriteLine("Dir Loc=" + DirectoryHelper.fileUploadDirectoryPath);
var zmodDir = new ZmodDirectory(DirectoryHelper.fileUploadDirectoryPath);
//loop of model
#region CODE - PMML
foreach (var item in zmodDir.PmmlFiles)
{
List <Property> _props = new List <Property>();
fileName = item.Value.info.Name;
fileExt = "pmml";
_url = DirectoryHelper.GetModelUrl(item.Value.info.Name);
//
ModelResponse newRecord = new ModelResponse()
{
Created_on = item.Value.info.CreationTime.ToString(),
Deployed = false,
Edited_on = item.Value.info.LastWriteTime.ToString(),
Extension = fileExt,
FilePath = item.Value.info.FullName,
Id = fileName.Replace($".{fileExt}", ""),
Loaded = false,
MimeType = "application/octet-stream",
Name = fileName,
Size = item.Value.info.Length,
Type = "PMML",
Url = _url,
User = "",
Properties = _props,
DateCreated = item.Value.info.CreationTime
};
ModelPayload.Create(newRecord);
}
#endregion
return(result);
}
public async Task <IActionResult> SavePmmlAsync(string id)
{
//variables
string response = string.Empty;
string reqBody = string.Empty;
string filePath = string.Empty;
string fileName = string.Empty;
string fileContent = string.Empty;
string fileUrl = string.Empty;
ModelResponse _data = new ModelResponse();
List <Property> _props = new List <Property>();
//read from request body
using (var reader = new StreamReader(Request.Body))
{
var body = reader.ReadToEnd();
reqBody = body.ToString();
}
//
JObject jObj = new JObject();
if (!string.IsNullOrEmpty(reqBody))
{
jObj = JObject.Parse(reqBody);
filePath = jObj["filePath"].ToString();
fileName = jObj["fileName"].ToString();
}
//Create blank pmml file
long fileSize = 0L;
string dirFullpath = DirectoryHelper.GetModelDirectoryPath();
string newFile = fileName;
try
{
//check if folder path exists...if not then create folder
if (!Directory.Exists(dirFullpath))
{
Directory.CreateDirectory(dirFullpath);
}
//call py service and add content to pmml file
fileContent = await autoMLclient.SaveBestPmml(filePath);
//
using (StreamWriter writer = new StreamWriter(Path.Combine(dirFullpath, newFile)))
{
await writer.WriteLineAsync(fileContent);
writer.Flush();
fileSize = writer.BaseStream.Length;
}
string _url = DirectoryHelper.GetModelUrl(newFile);
string _filePath = Path.Combine(dirFullpath, newFile);
//
ModelResponse newRecord = new ModelResponse()
{
Created_on = DateTime.Now.ToString(),
Deployed = false,
Edited_on = DateTime.Now.ToString(),
Extension = "pmml",
FilePath = _filePath,
Id = newFile.Replace($".pmml", ""),
Loaded = false,
MimeType = "application/octet-stream",
Name = newFile,
Size = fileSize,
Type = "PMML",
Url = _url,
User = CURRENT_USER,
Properties = _props
};
//
_data = ModelPayload.Create(newRecord);
}
catch (Exception ex)
{
//write to Ilogger
string message = ex.Message;
}
string jsonStr = JsonConvert.SerializeObject(_data, Formatting.None);
return(Json(_data));
}
public static bool ScanDirectoryToSeed()
{
bool result = false;
string fileName, _url, _fullName, fileContent, fileExt = "";
Console.WriteLine("Dir Loc=" + DirectoryHelper.fileUploadDirectoryPath);
var zmodDir = new ZmodDirectory(DirectoryHelper.fileUploadDirectoryPath);
//seed data - subdir, csv, img and json
#region DATA - SUBDIR
foreach (var subdir in Directory.GetDirectories(DirectoryHelper.GetDataDirectoryPath()))
{
string folderName = Path.GetFileName(subdir);
string _createdOn = Directory.GetCreationTime(subdir).ToString();
List <Property> _props = new List <Property>();
_props.Add(new Property {
key = "Subdirectories", value = DirectoryHelper.CountDirectories(subdir).ToString()
});
_props.Add(new Property {
key = "Files", value = DirectoryHelper.CountFiles(subdir).ToString()
});
DataResponse newRecord = new DataResponse()
{
Created_on = Directory.GetCreationTime(subdir).ToString(),
Edited_on = Directory.GetLastWriteTime(subdir).ToString(),
Extension = "",
Type = "FOLDER",
FilePath = subdir,
Id = folderName,
MimeType = "",
Name = folderName,
Properties = _props,
DateCreated = Directory.GetCreationTime(subdir)
};
//
DataPayload.Create(newRecord);
}
#endregion
#region DATA - CSV
foreach (var item in zmodDir.CsvFiles)
{
List <Property> _props = new List <Property>();
fileName = item.Value.info.Name;
fileExt = "csv";
_fullName = item.Value.info.FullName;
_fullName = _fullName.Substring(_fullName.IndexOf("Data")).Remove(0, 5);
_url = DirectoryHelper.GetDataUrl(_fullName);
//get properties row and column count
int[] csvProps = CsvHelper.GetCsvRowColumnCount(item.Value.info.FullName);
_props.Add(new Property {
key = "Number of Rows", value = csvProps[0].ToString()
});
_props.Add(new Property {
key = "Number of Columns", value = csvProps[1].ToString()
});
//
DataResponse newRecord = new DataResponse()
{
Created_on = item.Value.info.CreationTime.ToString(),
Edited_on = item.Value.info.LastWriteTime.ToString(),
Extension = fileExt,
FilePath = item.Value.info.FullName,
Id = fileName.Replace($".{fileExt}", ""),
MimeType = "text/csv",
Name = fileName,
Properties = _props,
Size = item.Value.info.Length,
Type = "CSV",
Url = _url,
User = "",
DateCreated = item.Value.info.CreationTime
};
//
DataPayload.Create(newRecord);
}
#endregion
#region DATA - IMAGES
foreach (var item in zmodDir.ImageFiles)
{
List <Property> _props = new List <Property>();
fileName = item.Value.info.Name;
fileExt = item.Value.info.Extension.Remove(0, 1);
_fullName = item.Value.info.FullName;
_fullName = _fullName.Substring(_fullName.IndexOf("Data")).Remove(0, 5);
_url = DirectoryHelper.GetDataUrl(_fullName).Replace("\\", "/");
//get properties
try
{
// using (var image = new Bitmap(System.Drawing.Image.FromFile(item.Value.info.FullName)))
using (var image = new Bitmap(item.Value.info.FullName))
{
_props.Add(new Property {
key = "Width", value = image.Width.ToString() + " px"
});
_props.Add(new Property {
key = "Height", value = image.Height.ToString() + " px"
});
image.Dispose();
}
//
DataResponse newRecord = new DataResponse()
{
Created_on = item.Value.info.CreationTime.ToString(),
Edited_on = item.Value.info.LastWriteTime.ToString(),
Extension = fileExt,
FilePath = item.Value.info.FullName,
Id = fileName.Replace($".{fileExt}", ""),
MimeType = $"image/{fileExt}",
Name = fileName,
Properties = _props,
Size = item.Value.info.Length,
Type = "IMAGE",
Url = _url,
User = "",
DateCreated = item.Value.info.CreationTime
};
//
DataPayload.Create(newRecord);
}
catch (Exception ex)
{
Console.WriteLine(ex.StackTrace);
}
}
#endregion
#region DATA - JSON
foreach (var item in zmodDir.JsonFiles)
{
List <Property> _props = new List <Property>();
fileName = item.Value.info.Name;
fileExt = "json";
_fullName = item.Value.info.FullName;
_fullName = _fullName.Substring(_fullName.IndexOf("Data")).Remove(0, 5);
_url = DirectoryHelper.GetDataUrl(_fullName);
fileContent = "";
//read json file from filestream
if (!string.IsNullOrEmpty(fileName))
{
using (StreamReader reader = new StreamReader(item.Value.info.FullName))
{
fileContent = reader.ReadToEnd();
}
}
//parse
try
{
if (!string.IsNullOrEmpty(fileContent))
{
JsonTextReader reader = new JsonTextReader(new StringReader(fileContent));
int objCtr = 0;
while (reader.Read())
{
if (reader.TokenType == JsonToken.EndObject)
{
objCtr++;
}
}
_props.Add(new Property {
key = "Number of Objects", value = objCtr.ToString()
});
//
DataResponse newRecord = new DataResponse()
{
Created_on = item.Value.info.CreationTime.ToString(),
Edited_on = item.Value.info.LastWriteTime.ToString(),
Extension = fileExt,
FilePath = item.Value.info.FullName,
Id = fileName.Replace($".{fileExt}", ""),
MimeType = "application/json",
Name = fileName,
Properties = _props,
Size = item.Value.info.Length,
Type = "JSON",
Url = _url,
User = "",
DateCreated = item.Value.info.CreationTime
};
//
DataPayload.Create(newRecord);
}
}
catch (Exception ex)
{
//TODO: logger
string err = ex.StackTrace;
}
}
#endregion
#region DATA - MP4
foreach (var item in zmodDir.VideoFiles)
{
List <Property> _props = new List <Property>();
fileName = item.Value.info.Name;
fileExt = item.Value.info.Extension.Remove(0, 1);
_fullName = item.Value.info.FullName;
_fullName = _fullName.Substring(_fullName.IndexOf("Data")).Remove(0, 5);
_url = DirectoryHelper.GetDataUrl(_fullName);
//get properties
try
{
DataResponse newRecord = new DataResponse()
{
Created_on = item.Value.info.CreationTime.ToString(),
Edited_on = item.Value.info.LastWriteTime.ToString(),
Extension = fileExt,
FilePath = item.Value.info.FullName,
Id = fileName.Replace($".{fileExt}", ""),
MimeType = $"video/{fileExt}",
Name = fileName,
Properties = _props,
Size = item.Value.info.Length,
Type = "VIDEO",
Url = _url,
User = "",
DateCreated = item.Value.info.CreationTime
};
//
DataPayload.Create(newRecord);
}
catch (Exception ex)
{
//TODO: logger
string err = ex.StackTrace;
}
}
#endregion
#region DATA - TEXT
foreach (var item in zmodDir.TextFiles)
{
List <Property> _props = new List <Property>();
fileName = item.Value.info.Name;
fileExt = item.Value.info.Extension.Remove(0, 1);
_fullName = item.Value.info.FullName;
_fullName = _fullName.Substring(_fullName.IndexOf("Data")).Remove(0, 5);
_url = DirectoryHelper.GetDataUrl(_fullName);
//get properties
try
{
DataResponse newRecord = new DataResponse()
{
Created_on = item.Value.info.CreationTime.ToString(),
Edited_on = item.Value.info.LastWriteTime.ToString(),
Extension = fileExt,
FilePath = item.Value.info.FullName,
Id = fileName.Replace($".{fileExt}", ""),
MimeType = $"text/plain",
Name = fileName,
Properties = _props,
Size = item.Value.info.Length,
Type = "TEXT",
Url = _url,
User = "",
DateCreated = item.Value.info.CreationTime
};
//
DataPayload.Create(newRecord);
}
catch (Exception ex)
{
Debug.WriteLine(ex.StackTrace);
}
}
#endregion
//seed code - py and ipynb
#region CODE - PY
foreach (var item in zmodDir.PyFiles)
{
fileName = item.Value.info.Name;
fileExt = "py";
_url = DirectoryHelper.GetCodeUrl(item.Value.info.Name);
CodeResponse newRecord = new CodeResponse()
{
Created_on = item.Value.info.CreationTime.ToString(),
Edited_on = item.Value.info.LastWriteTime.ToString(),
Extension = fileExt,
FilePath = item.Value.info.FullName,
Id = fileName.Replace($".{fileExt}", ""),
MimeType = "application/octet-stream",
Name = fileName,
Size = item.Value.info.Length,
Type = "PYTHON",
Url = _url,
User = "",
DateCreated = item.Value.info.CreationTime
};
CodePayload.Create(newRecord);
}
#endregion
#region CODE - IPYNB
foreach (var item in zmodDir.IpynbFiles)
{
fileName = item.Value.info.Name;
fileExt = Path.GetExtension(fileName).Remove(0, 1);
_url = DirectoryHelper.GetCodeUrl(item.Value.info.Name);
CodeResponse newRecord = new CodeResponse()
{
Created_on = item.Value.info.CreationTime.ToString(),
Edited_on = item.Value.info.LastWriteTime.ToString(),
Extension = fileExt,
FilePath = item.Value.info.FullName,
Id = fileName.Replace($".{fileExt}", ""),
MimeType = "application/octet-stream",
Name = fileName,
Size = item.Value.info.Length,
Type = "JUPYTER_NOTEBOOK",
Url = _url,
User = "",
DateCreated = item.Value.info.CreationTime
};
CodePayload.Create(newRecord);
}
#endregion
#region DATA - R
foreach (var item in zmodDir.RFiles)
{
fileName = item.Value.info.Name;
fileExt = "r";
_url = DirectoryHelper.GetCodeUrl(item.Value.info.Name);
CodeResponse newRecord = new CodeResponse()
{
Created_on = item.Value.info.CreationTime.ToString(),
Edited_on = item.Value.info.LastWriteTime.ToString(),
Extension = fileExt,
FilePath = item.Value.info.FullName,
Id = Path.GetFileNameWithoutExtension(fileName),
MimeType = "application/octet-stream",
Name = fileName,
Size = item.Value.info.Length,
Type = "R",
Url = _url,
User = "",
DateCreated = item.Value.info.CreationTime
};
CodePayload.Create(newRecord);
}
#endregion
//loop of model
#region MODEL - PMML
foreach (var item in zmodDir.PmmlFiles)
{
List <Property> _props = new List <Property>();
fileName = item.Value.info.Name;
fileExt = "pmml";
_url = DirectoryHelper.GetModelUrl(item.Value.info.Name);
//
ModelResponse newRecord = new ModelResponse()
{
Created_on = item.Value.info.CreationTime.ToString(),
Deployed = false,
Edited_on = item.Value.info.LastWriteTime.ToString(),
Extension = fileExt,
FilePath = item.Value.info.FullName,
Id = fileName.Replace($".{fileExt}", ""),
Loaded = false,
MimeType = "application/octet-stream",
Name = fileName,
Size = item.Value.info.Length,
Type = "PMML",
Url = _url,
User = "",
Properties = _props,
DateCreated = item.Value.info.CreationTime
};
ModelPayload.Create(newRecord);
}
#endregion
#region MODEL - H5
foreach (var item in zmodDir.H5Files)
{
List <Property> _props = new List <Property>();
fileName = item.Value.info.Name;
fileExt = "h5";
_url = DirectoryHelper.GetModelUrl(item.Value.info.Name);
//
ModelResponse newRecord = new ModelResponse()
{
Created_on = item.Value.info.CreationTime.ToString(),
Deployed = false,
Edited_on = item.Value.info.LastWriteTime.ToString(),
Extension = fileExt,
FilePath = item.Value.info.FullName,
Id = fileName.Replace($".{fileExt}", ""),
Loaded = false,
MimeType = "application/octet-stream",
Name = fileName,
Size = item.Value.info.Length,
Type = "H5",
Url = _url,
User = "",
Properties = _props,
DateCreated = item.Value.info.CreationTime
};
ModelPayload.Create(newRecord);
}
#endregion
return(result);
}
public async Task <IActionResult> Post(List <IFormFile> file)
{
#region variables
List <ModelResponse> _response = new List <ModelResponse>();
List <ModelResponse> existingCodeData = new List <ModelResponse>();
long size = file.Sum(f => f.Length);
string type = string.Empty;
bool IsFileExists = false;
// full path to file in temp location
var filePath = Path.GetTempFileName();
string dirFullpath = DirectoryHelper.GetModelDirectoryPath();
#endregion
//check if folder path exists...if not then create folder
if (!Directory.Exists(dirFullpath))
{
Directory.CreateDirectory(dirFullpath);
}
foreach (var formFile in file)
{
if (formFile.Length > 0)
{
//check if the file with the same name exists
existingCodeData = ModelPayload.Get();
if (existingCodeData.Count > 0)
{
//
foreach (var record in existingCodeData)
{
if ((record.Name == formFile.FileName) && (record.User == CURRENT_USER))
{
IsFileExists = true;
}
}
}
existingCodeData.Clear();
//
if (!IsFileExists)
{
string fileExt = System.IO.Path.GetExtension(formFile.FileName).Substring(1).ToString().ToLower();
// upload file start
using (var fileStream = new FileStream(Path.Combine(dirFullpath, formFile.FileName), FileMode.Create))
{
//check file allowed extensions
if (!extensions.Contains(fileExt))
{
return(BadRequest("File type not allowed"));
}
else
{
await formFile.CopyToAsync(fileStream);
}
}
if (fileExt.Contains("pmml"))
{
type = "PMML";
}
List <Property> _props = new List <Property>();
string _url = DirectoryHelper.GetModelUrl(formFile.FileName);
string _filePath = Path.Combine(dirFullpath, formFile.FileName);
//
ModelResponse newRecord = new ModelResponse()
{
Created_on = DateTime.Now.ToString(),
Deployed = false,
Edited_on = DateTime.Now.ToString(),
Extension = fileExt,
FilePath = _filePath,
Id = formFile.FileName.Replace($".{fileExt}", ""),
Loaded = false,
MimeType = formFile.ContentType,
Name = formFile.FileName,
Size = formFile.Length,
Type = type,
Url = _url,
User = CURRENT_USER,
Properties = _props
};
//
_response.Add(ModelPayload.Create(newRecord));
}
}
}
return(Ok(_response));
}
public async Task <IActionResult> CreatePmmlAsync()
{
//create blank pmml file
long fileSize = 0L;
string dirFullpath = DirectoryHelper.GetModelDirectoryPath();
string newFile = "New_" + DateTime.Now.Ticks.ToString() + ".pmml";
string _filePath = Path.Combine(dirFullpath, newFile);
ModelResponse _data = new ModelResponse();
List <Property> _props = new List <Property>();
try
{
//check if folder path exists...if not then create folder
if (!Directory.Exists(dirFullpath))
{
Directory.CreateDirectory(dirFullpath);
}
//create blank model data
StringBuilder fileContent = new StringBuilder();
fileContent.Append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>,@,");
fileContent.Append("<PMML xmlns=\"http://www.dmg.org/PMML-4_3\" version=\"4.3Ext\">,@,");
fileContent.Append("<Header copyright=\"Copyright (c) 2018 Software AG\" description=\"Neural Network Model\">,@,");
fileContent.Append("<Timestamp>" + DateTime.Now.ToString("yyyy-MM-dd H:mm:ss.") + TimeSpan.TicksPerMillisecond + "</Timestamp>,@,");
fileContent.Append("</Header>,@,");
fileContent.Append("<DeepNetwork>,@,");
fileContent.Append("</DeepNetwork>,@,");
fileContent.Append("</PMML>");
//
using (StreamWriter writer = new StreamWriter(_filePath))
{
foreach (string line in fileContent.ToString().Split(",@,"))
{
writer.WriteLine(line);
}
writer.Flush();
fileSize = writer.BaseStream.Length;
}
string _url = DirectoryHelper.GetModelUrl(newFile);
//
ModelResponse newRecord = new ModelResponse()
{
Created_on = DateTime.Now.ToString(),
Deployed = false,
Edited_on = DateTime.Now.ToString(),
Extension = "pmml",
FilePath = _filePath,
Id = newFile.Replace($".pmml", ""),
Loaded = false,
MimeType = "application/octet-stream",
Name = newFile,
Size = fileSize,
Type = "PMML",
Url = _url,
User = CURRENT_USER,
Properties = _props
};
//
_data = ModelPayload.Create(newRecord);
await Task.FromResult(0);
}
catch (Exception ex)
{
//write to Ilogger
string message = ex.Message;
}
string jsonStr = JsonConvert.SerializeObject(_data, Formatting.None);
return(Json(_data));
}
public async Task <IActionResult> ModifyFilenameAsync(string id)
{
string newFileName = "";
string reqBody = "";
await System.Threading.Tasks.Task.FromResult(0);
try
{
//read request body
using (var reader = new StreamReader(Request.Body))
{
var body = reader.ReadToEnd();
reqBody = body.ToString();
}
//get new filename
if (!string.IsNullOrEmpty(reqBody))
{
var content = JObject.Parse(reqBody);
newFileName = (string)content["newName"];
newFileName = Regex.Replace(newFileName, "[\n\r\t]", string.Empty);
newFileName = Regex.Replace(newFileName, @"\s", string.Empty);
}
if (!string.IsNullOrEmpty(newFileName))
{
//if same name exist - BadRequest
foreach (var record in responseData)
{
if (record.Id.ToLower() == newFileName.ToLower())
{
return(BadRequest(new { message = "File with same name already exists." }));
}
}
//rename the file and/or folder
foreach (var record in responseData)
{
if (record.Id.ToString() == id)
{
var newfilePath = record.FilePath.Replace($"{id}.{record.Extension}", $"{newFileName}.{record.Extension}");
FileFolderHelper.RenameFile(record.FilePath, newfilePath);
var newRecord = new ModelResponse()
{
Created_on = record.Created_on,
Edited_on = record.Edited_on,
Extension = record.Extension,
FilePath = newfilePath,
Id = newFileName,
MimeType = record.MimeType,
Name = $"{newFileName}.{record.Extension}",
Properties = record.Properties,
Size = record.Size,
Type = record.Type,
Url = record.Url.Replace(id, newFileName),
User = record.User
};
ModelPayload.Create(newRecord);
ModelPayload.RemoveOnlyFromModelPayload(id);
return(Json(newRecord));
}
}
}
}
catch (Exception ex)
{
return(BadRequest(new { message = "Renaming file failed.", exception = ex.StackTrace }));
}
return(BadRequest(new { message = "Renaming file failed." }));
}
