public static bool RemoveCompleted(string id) { //id is the filename bool result = false; FilesInProgress _data = new FilesInProgress(); result = GlobalStorage.FilesInProgressStorage.TryRemove(id, out _data); return(result); }
public static bool Clear(string id) { bool result = false; FilesInProgress _data = new FilesInProgress(); foreach (var item in GlobalStorage.FilesInProgressStorage) { GlobalStorage.FilesInProgressStorage.TryRemove(id, out _data); result = true; } return(result); }
public async Task <string> GetFile(byte[] fileHash) { var requestMessage = new GetFileMessage(fileHash); var subDir = new DirectoryInfo(Path.Combine(TempStorageDirectory, Hasher.GetDirectoryNameFromHash(fileHash))); if (!subDir.Exists) { subDir.Create(); } var fileName = Hasher.GetFileNameFromHash(fileHash); var filePath = Path.Combine(subDir.FullName, fileName); FilesInProgress.Add(fileHash, File.OpenWrite(filePath)); await Communicator.SendMessage(requestMessage); // wait for transfer var gotVerificationMessage = false; string resultsPath = null; while (!gotVerificationMessage) { if (VerificationMessages.ContainsKey(fileHash)) { var verifyMessage = VerificationMessages[fileHash] as FileStoredVerificationMessage; resultsPath = (verifyMessage?.Success ?? false) ? verifyMessage.FilePath : null; VerificationMessages.Remove(fileHash); gotVerificationMessage = true; } await Task.Delay(1); } return(resultsPath); }
public static FilesInProgress Create(FilesInProgress newRecord) { GlobalStorage.FilesInProgressStorage.TryAdd(newRecord.Id, newRecord); return(newRecord); }
public async Task <IActionResult> Post(List <IFormFile> file) { List <CodeResponse> response = new List <CodeResponse>(); List <CodeResponse> existingCodeData = new List <CodeResponse>(); long size = file.Sum(f => f.Length); bool IsFileExists = false; string type, dirFullpath, nbPath, _url = string.Empty; // full path to file in temp location var filePath = Path.GetTempFileName(); foreach (var formFile in file) { dirFullpath = DirectoryHelper.GetCodeDirectoryPath(); #region type var fileExt = System.IO.Path.GetExtension(formFile.FileName).Substring(1); // upload file edn and start creating payload switch (fileExt.ToLower()) { case "py": type = "PYTHON"; break; case "ipynb": type = "JUPYTER_NOTEBOOK"; break; case "r": type = "R"; break; default: type = "UNRECOGNIZED"; break; } #endregion if (formFile.Length > 0) { //check if the file with the same name exists existingCodeData = CodePayload.Get(); if (existingCodeData.Count > 0) { // foreach (var record in existingCodeData) { if (record.Name == formFile.FileName) { IsFileExists = true; } } } existingCodeData.Clear(); // if (!FilePathHelper.IsFileNameValid(formFile.FileName)) { return(BadRequest("File name not valid.")); } if (!IsFileExists) { #region upload large file > 400MB if (size > 40000000) { Console.WriteLine(">>>>>>>>>>>>>>>>>>>>UPLOADING LARGE CODE FILE................................"); #region add to uploading // FilesInProgress wip = new FilesInProgress() { Id = formFile.FileName, CreatedAt = DateTime.Now, Name = formFile.FileName, Type = type, Module = "CODE", UploadStatus = "INPROGRESS" }; FilesUploadingPayload.Create(wip); #endregion //check if same job is scheduled ISchedulerFactory schfack = new StdSchedulerFactory(); IScheduler scheduler = await schfack.GetScheduler(); var jobKey = new JobKey(filePath); if (await scheduler.CheckExists(jobKey)) { await scheduler.ResumeJob(jobKey); } else { #region create quartz job for training model ITrigger trigger = TriggerBuilder.Create() .WithIdentity($"Uploading Code file-{DateTime.Now}") .WithPriority(1) .Build(); IJobDetail job = JobBuilder.Create <UploadJob>() .WithIdentity(filePath) .Build(); job.JobDataMap["id"] = formFile.FileName.Replace($".{fileExt}", ""); job.JobDataMap["filePath"] = filePath; await _scheduler.ScheduleJob(job, trigger); #endregion } } #endregion if (fileExt.ToLower() == "ipynb") { dirFullpath = dirFullpath + formFile.FileName.Replace(".ipynb", "/"); _url = DirectoryHelper.GetCodeUrl(formFile.FileName.Replace(".ipynb", string.Empty) + "/" + formFile.FileName); } else { _url = DirectoryHelper.GetCodeUrl(formFile.FileName); } nbPath = dirFullpath + formFile.FileName; //check if folder path exists...if not then create folder if (!Directory.Exists(dirFullpath)) { Directory.CreateDirectory(dirFullpath); } // upload file start using (var fileStream = new FileStream(Path.Combine(dirFullpath, formFile.FileName), FileMode.Create)) { //check file allowed extensions if (!extensions.Contains(fileExt.ToString().ToLower())) { return(BadRequest("File type not allowed")); } //file copy await formFile.CopyToAsync(fileStream); CodeResponse newRecord = new CodeResponse() { Id = formFile.FileName.Replace("." + fileExt, ""), Name = formFile.FileName, User = "", Created_on = DateTime.Now.ToString(), Edited_on = DateTime.Now.ToString(), Extension = fileExt, MimeType = formFile.ContentType, Size = formFile.Length, Type = type, Url = _url, FilePath = nbPath, DateCreated = DateTime.Now }; CodePayload.Create(newRecord); response.Add(newRecord); } } else { return(Conflict(new { message = "File already exists.", error = "File already exists." })); } } IsFileExists = false; } if (response.Count > 0) { return(Ok(response)); } else { return(BadRequest("File already exists.")); } }
protected void ProcessMessage(object sender, MessageReceivedArgs e) { switch (e.Message) { case FileStartMessage message: if (!FilesInProgress.ContainsKey(message.Payload)) { var fileHash = message.Payload; var subDir = new DirectoryInfo(Path.Combine(TempStorageDirectory, Hasher.GetDirectoryNameFromHash(fileHash))); if (!subDir.Exists) { subDir.Create(); } var fileName = Hasher.GetFileNameFromHash(fileHash); var filePath = Path.Combine(subDir.FullName, fileName); FilesInProgress.Add(message.Payload, File.OpenWrite(filePath)); } break; case FileChunckMessage message: if (FilesInProgress.ContainsKey(message.FileHash)) { var stream = FilesInProgress[message.FileHash]; stream.Write(message.FileData, 0, message.FileData.Length); var receipt = new FileChunkReceivedMessage(message.FileHash, message.DataChecksum); Communicator.SendMessage(((INetworkClient)sender).ClientName, receipt); } break; case FileEndMessage message: if (FilesInProgress.ContainsKey(message.Payload)) { var stream = FilesInProgress[message.Payload]; stream.Flush(); var fileHash = message.Payload; var subDir = Hasher.GetDirectoryNameFromHash(fileHash); var fileName = Hasher.GetFileNameFromHash(fileHash); var filePath = Path.Combine(TempStorageDirectory, subDir, fileName); stream.Dispose(); ReadFile(((INetworkClient)sender).ClientName, message.Payload, new FileInfo(filePath)); FilesInProgress.Remove(message.Payload); } break; case AbandonFileTransferMessage message: if (FilesInProgress.ContainsKey(message.Payload)) { var stream = FilesInProgress[message.Payload]; stream.Dispose(); FilesInProgress.Remove(message.Payload); } break; case FileChunkReceivedMessage message: FileTransferResponse[message.FileHash] = true; break; case FileStoredVerificationMessage message: VerificationMessages.Add(message.FileHash, message); break; case GetFileMessage message: SendFile(((INetworkClient)sender).ClientName, message.Payload); break; } }