public static bool RemoveCompleted(string id)
{
//id is the filename
bool result = false;
FilesInProgress _data = new FilesInProgress();
result = GlobalStorage.FilesInProgressStorage.TryRemove(id, out _data);
return(result);
}
public static bool Clear(string id)
{
bool result = false;
FilesInProgress _data = new FilesInProgress();
foreach (var item in GlobalStorage.FilesInProgressStorage)
{
GlobalStorage.FilesInProgressStorage.TryRemove(id, out _data);
result = true;
}
return(result);
}
public async Task <string> GetFile(byte[] fileHash)
{
var requestMessage = new GetFileMessage(fileHash);
var subDir = new DirectoryInfo(Path.Combine(TempStorageDirectory, Hasher.GetDirectoryNameFromHash(fileHash)));
if (!subDir.Exists)
{
subDir.Create();
}
var fileName = Hasher.GetFileNameFromHash(fileHash);
var filePath = Path.Combine(subDir.FullName, fileName);
FilesInProgress.Add(fileHash, File.OpenWrite(filePath));
await Communicator.SendMessage(requestMessage);
// wait for transfer
var gotVerificationMessage = false;
string resultsPath = null;
while (!gotVerificationMessage)
{
if (VerificationMessages.ContainsKey(fileHash))
{
var verifyMessage = VerificationMessages[fileHash] as FileStoredVerificationMessage;
resultsPath = (verifyMessage?.Success ?? false) ? verifyMessage.FilePath : null;
VerificationMessages.Remove(fileHash);
gotVerificationMessage = true;
}
await Task.Delay(1);
}
return(resultsPath);
}
public static FilesInProgress Create(FilesInProgress newRecord)
{
GlobalStorage.FilesInProgressStorage.TryAdd(newRecord.Id, newRecord);
return(newRecord);
}
public async Task <IActionResult> Post(List <IFormFile> file)
{
List <CodeResponse> response = new List <CodeResponse>();
List <CodeResponse> existingCodeData = new List <CodeResponse>();
long size = file.Sum(f => f.Length);
bool IsFileExists = false;
string type, dirFullpath, nbPath, _url = string.Empty;
// full path to file in temp location
var filePath = Path.GetTempFileName();
foreach (var formFile in file)
{
dirFullpath = DirectoryHelper.GetCodeDirectoryPath();
#region type
var fileExt = System.IO.Path.GetExtension(formFile.FileName).Substring(1);
// upload file edn and start creating payload
switch (fileExt.ToLower())
{
case "py":
type = "PYTHON";
break;
case "ipynb":
type = "JUPYTER_NOTEBOOK";
break;
case "r":
type = "R";
break;
default:
type = "UNRECOGNIZED";
break;
}
#endregion
if (formFile.Length > 0)
{
//check if the file with the same name exists
existingCodeData = CodePayload.Get();
if (existingCodeData.Count > 0)
{
//
foreach (var record in existingCodeData)
{
if (record.Name == formFile.FileName)
{
IsFileExists = true;
}
}
}
existingCodeData.Clear();
//
if (!FilePathHelper.IsFileNameValid(formFile.FileName))
{
return(BadRequest("File name not valid."));
}
if (!IsFileExists)
{
#region upload large file > 400MB
if (size > 40000000)
{
Console.WriteLine(">>>>>>>>>>>>>>>>>>>>UPLOADING LARGE CODE FILE................................");
#region add to uploading
//
FilesInProgress wip = new FilesInProgress()
{
Id = formFile.FileName,
CreatedAt = DateTime.Now,
Name = formFile.FileName,
Type = type,
Module = "CODE",
UploadStatus = "INPROGRESS"
};
FilesUploadingPayload.Create(wip);
#endregion
//check if same job is scheduled
ISchedulerFactory schfack = new StdSchedulerFactory();
IScheduler scheduler = await schfack.GetScheduler();
var jobKey = new JobKey(filePath);
if (await scheduler.CheckExists(jobKey))
{
await scheduler.ResumeJob(jobKey);
}
else
{
#region create quartz job for training model
ITrigger trigger = TriggerBuilder.Create()
.WithIdentity($"Uploading Code file-{DateTime.Now}")
.WithPriority(1)
.Build();
IJobDetail job = JobBuilder.Create <UploadJob>()
.WithIdentity(filePath)
.Build();
job.JobDataMap["id"] = formFile.FileName.Replace($".{fileExt}", "");
job.JobDataMap["filePath"] = filePath;
await _scheduler.ScheduleJob(job, trigger);
#endregion
}
}
#endregion
if (fileExt.ToLower() == "ipynb")
{
dirFullpath = dirFullpath + formFile.FileName.Replace(".ipynb", "/");
_url = DirectoryHelper.GetCodeUrl(formFile.FileName.Replace(".ipynb", string.Empty) + "/" + formFile.FileName);
}
else
{
_url = DirectoryHelper.GetCodeUrl(formFile.FileName);
}
nbPath = dirFullpath + formFile.FileName;
//check if folder path exists...if not then create folder
if (!Directory.Exists(dirFullpath))
{
Directory.CreateDirectory(dirFullpath);
}
// upload file start
using (var fileStream = new FileStream(Path.Combine(dirFullpath, formFile.FileName), FileMode.Create))
{
//check file allowed extensions
if (!extensions.Contains(fileExt.ToString().ToLower()))
{
return(BadRequest("File type not allowed"));
}
//file copy
await formFile.CopyToAsync(fileStream);
CodeResponse newRecord = new CodeResponse()
{
Id = formFile.FileName.Replace("." + fileExt, ""),
Name = formFile.FileName,
User = "",
Created_on = DateTime.Now.ToString(),
Edited_on = DateTime.Now.ToString(),
Extension = fileExt,
MimeType = formFile.ContentType,
Size = formFile.Length,
Type = type,
Url = _url,
FilePath = nbPath,
DateCreated = DateTime.Now
};
CodePayload.Create(newRecord);
response.Add(newRecord);
}
}
else
{
return(Conflict(new { message = "File already exists.", error = "File already exists." }));
}
}
IsFileExists = false;
}
if (response.Count > 0)
{
return(Ok(response));
}
else
{
return(BadRequest("File already exists."));
}
}
protected void ProcessMessage(object sender, MessageReceivedArgs e)
{
switch (e.Message)
{
case FileStartMessage message:
if (!FilesInProgress.ContainsKey(message.Payload))
{
var fileHash = message.Payload;
var subDir = new DirectoryInfo(Path.Combine(TempStorageDirectory, Hasher.GetDirectoryNameFromHash(fileHash)));
if (!subDir.Exists)
{
subDir.Create();
}
var fileName = Hasher.GetFileNameFromHash(fileHash);
var filePath = Path.Combine(subDir.FullName, fileName);
FilesInProgress.Add(message.Payload, File.OpenWrite(filePath));
}
break;
case FileChunckMessage message:
if (FilesInProgress.ContainsKey(message.FileHash))
{
var stream = FilesInProgress[message.FileHash];
stream.Write(message.FileData, 0, message.FileData.Length);
var receipt = new FileChunkReceivedMessage(message.FileHash, message.DataChecksum);
Communicator.SendMessage(((INetworkClient)sender).ClientName, receipt);
}
break;
case FileEndMessage message:
if (FilesInProgress.ContainsKey(message.Payload))
{
var stream = FilesInProgress[message.Payload];
stream.Flush();
var fileHash = message.Payload;
var subDir = Hasher.GetDirectoryNameFromHash(fileHash);
var fileName = Hasher.GetFileNameFromHash(fileHash);
var filePath = Path.Combine(TempStorageDirectory, subDir, fileName);
stream.Dispose();
ReadFile(((INetworkClient)sender).ClientName, message.Payload, new FileInfo(filePath));
FilesInProgress.Remove(message.Payload);
}
break;
case AbandonFileTransferMessage message:
if (FilesInProgress.ContainsKey(message.Payload))
{
var stream = FilesInProgress[message.Payload];
stream.Dispose();
FilesInProgress.Remove(message.Payload);
}
break;
case FileChunkReceivedMessage message:
FileTransferResponse[message.FileHash] = true;
break;
case FileStoredVerificationMessage message:
VerificationMessages.Add(message.FileHash, message);
break;
case GetFileMessage message:
SendFile(((INetworkClient)sender).ClientName, message.Payload);
break;
}
}
