public IActionResult Add([FromRoute] long fileId, [FromBody] FileBlockInfo fileBlockInfo) { FileBlock fileBlock = _db.FileBlocks.FirstOrDefault(fb => fb.FileId == fileId && fb.Number == fileBlockInfo.Number); using (MD5 md5Hash = MD5.Create()) { byte[] rawContent = Convert.FromBase64String(fileBlockInfo.Value); if (fileBlock == null) //Create new fileBlock { _db.FileBlocks.Add(new FileBlock() { FileId = fileId, Number = fileBlockInfo.Number, Content = rawContent, Checksum = MD5Utils.GetMd5Hash(md5Hash, rawContent.ToString()) }); } else //Update existing fileBlock { fileBlock.Content = rawContent; fileBlock.Checksum = MD5Utils.GetMd5Hash(md5Hash, rawContent.ToString()); _db.FileBlocks.Update(fileBlock); } _db.SaveChanges(); } return(StatusCode((int)HttpStatusCode.OK)); }
public FileBlockInfo ClacFileBlockInfo(string path, FileStream fs, int blockSize = 8192) { long fileLength = fs.Length; if (fileLength <= 0) { throw new ArgumentOutOfRangeException("文件长度不应为0"); } long position = fs.Position; long blockCountF = fileLength / blockSize; int lastBlockSize = (int)(fileLength - blockCountF * blockSize); var blockCount = lastBlockSize <= 0 ? (int)blockCountF : (int)blockCountF + 1; if (fileLength < blockSize) { blockCount = 1; lastBlockSize = (int)fileLength; blockSize = lastBlockSize; } var info = new FileBlockInfo() { BlockCount = blockCount, BlockSize = blockSize, LastBlockSize = lastBlockSize, FileName = Path.GetFileName(path), FileMd5 = Md5.GetMd5WithFileStream(fs, fs.Position), FileLength = fileLength }; fs.Position = position; return(info); }
private FileTransferRequest GenerateTransferRequest(FileBlockInfo info) { int seed = GetRandomSeed(); var random = new Random(seed); return(GenerateTransferRequest(info, random.Next(int.MinValue, int.MaxValue))); }
/// <summary> /// 计算每个文件块的MD5值 /// </summary> /// <param name="info"></param> /// <param name="fs"></param> /// <returns></returns> private List <BlockTransferRequest> ClacFileEachBlockMd5(FileBlockInfo info, FileStream fs) { var position = fs.Position; var list = new List <BlockTransferRequest>(); int sendBlockSize = info.BlockSize; int blockIndex = 0; while (fs.Position < fs.Length) { var r = new BlockTransferRequest() { FileName = info.FileName, BlockIndex = blockIndex, BlockSize = sendBlockSize, SeekOffset = (int)fs.Position }; byte[] buffer = new byte[sendBlockSize]; fs.Read(buffer, 0, buffer.Length);//read之后,Position会变,所以要先生成对象 r.BlockMd5 = Md5.GetMd5WithBytes(buffer); list.Add(r); blockIndex++;//增加索引序号,别漏了 if (sendBlockSize > fs.Length - fs.Position) { sendBlockSize = (int)(fs.Length - fs.Position); } } fs.Position = position; return(list); }
public async Task <IActionResult> UploadFile() { foreach (var file in Request.Form.Files) { if (file.Length == 0) { continue; } using (var contentStream = file.OpenReadStream()) { var fileBlockInfo = new FileBlockInfo { FileId = Request.Form["flowIdentifier"], BlockId = Request.Form["flowIdentifier"], FileName = Request.Form["flowFilename"], FileSize = long.Parse(Request.Form["flowTotalSize"]), BlockSize = long.Parse(Request.Form["flowChunkSize"]), SequenceNum = int.Parse(Request.Form["flowChunkNumber"]), TotalBlocksCount = int.Parse(Request.Form["flowTotalChunks"]) }; var uploadToAzure = Convert.ToBoolean(Request.Form["toAzure"]); await _fileUploadManager.AddFileBlockAsync(_serviceProvider, fileBlockInfo, contentStream, uploadToAzure); } } return(Ok()); }
public bool SetFileBlockContent(long fileId, FileBlockInfo fileBlockInfo) { string route = $"{BaseAddress}/File/{fileId}/FileBlock"; var result = QueryRequest(route, HttpMethod.Post, new StringContent(JsonConvert.SerializeObject(fileBlockInfo), Encoding.UTF8, "application/json")); if (result.status != HttpStatusCode.OK) { return(false); } return(true); }
public FileTransferRequest GenerateTransferRequest(FileBlockInfo info, int randomSeed) { var request = new FileTransferRequest() { BlockIndex = 0, BlockCount = info.BlockCount, LastBlockSize = info.LastBlockSize, EachBlockSize = info.BlockSize, FileMd5 = info.FileMd5, FileName = info.FileName, FileSuffix = Path.GetExtension(info.FileName), FileSize = info.FileLength, IsSendingOver = false, SeekOffset = 0, BlockData = null, RequestId = randomSeed }; return(request); }
/// <summary> /// 计算文件块索引 /// </summary> /// <param name="info"></param> /// <param name="remotePosition"></param> /// <param name="realBlockIndex"></param> /// <param name="realFsPosition"></param> private void ClacBlockIndex(FileBlockInfo info, long remotePosition, out int realBlockIndex, out long realFsPosition) { var realIndex = (int)(remotePosition / info.BlockSize); var realPosition = info.BlockSize * realIndex; if (realPosition + info.LastBlockSize == info.FileLength)//判断是否为倒数第二个Block { realBlockIndex = info.BlockCount - 1; realFsPosition = info.FileLength - info.LastBlockSize; } else if (remotePosition == info.FileLength)//判断是否为最后一个Block { realBlockIndex = info.BlockCount; realFsPosition = info.FileLength; } else { realBlockIndex = realPosition; realFsPosition = realIndex; } }
/// <summary> /// 根据反馈结果,重发文件 /// </summary> /// <param name="path"></param> /// <param name="fs"></param> /// <param name="fileBlockInfo"></param> /// <param name="request"></param> /// <param name="checkResponsed"></param> public bool ReSendFile(string path, FileStream fs, FileBlockInfo fileBlockInfo, FileTransferRequest request, FileTransferResponsed checkResponsed) { int resendCountOut = 0; while (!checkResponsed.FileMd5CheckResult && resendCountOut < 3)//最多重发三次 { #if DEBUG Console.WriteLine("ReSending File, count " + resendCountOut); #endif //MD5检查失败重发 var blockMsgList = ClacFileEachBlockMd5(fileBlockInfo, fs); if (blockMsgList.Count != fileBlockInfo.BlockCount) { throw new Exception("校准MD5时,Block数量计算错误!"); } int size = request.EachBlockSize; request.IsSendingOver = false; for (int i = 0; i < blockMsgList.Count; i++) { var blockCheck = _sendAdapter.UpdateFileBlockMessage(blockMsgList[i]); if (blockCheck.IsError) { if (i == blockMsgList.Count - 1) { size = request.LastBlockSize; } request.BlockIndex = i; request.SeekOffset = i * fileBlockInfo.BlockSize; request.BlockData = ReadFileBytes(path, request.SeekOffset, size); _sendAdapter.UpdateFileData(request); } } request.IsSendingOver = true; checkResponsed = _sendAdapter.UpdateFileData(request); resendCountOut++; } return(checkResponsed.FileMd5CheckResult); }
public void Read(TProtocol iprot) { iprot.IncrementRecursionDepth(); try { TField field; iprot.ReadStructBegin(); while (true) { field = iprot.ReadFieldBegin(); if (field.Type == TType.Stop) { break; } switch (field.ID) { case 1: if (field.Type == TType.I64) { FileId = iprot.ReadI64(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 2: if (field.Type == TType.String) { Name = iprot.ReadString(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 3: if (field.Type == TType.String) { Path = iprot.ReadString(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 4: if (field.Type == TType.String) { UfsPath = iprot.ReadString(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 5: if (field.Type == TType.I64) { Length = iprot.ReadI64(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 6: if (field.Type == TType.I64) { BlockSizeBytes = iprot.ReadI64(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 7: if (field.Type == TType.I64) { CreationTimeMs = iprot.ReadI64(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 8: if (field.Type == TType.Bool) { Completed = iprot.ReadBool(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 9: if (field.Type == TType.Bool) { Folder = iprot.ReadBool(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 10: if (field.Type == TType.Bool) { Pinned = iprot.ReadBool(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 11: if (field.Type == TType.Bool) { Cacheable = iprot.ReadBool(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 12: if (field.Type == TType.Bool) { Persisted = iprot.ReadBool(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 13: if (field.Type == TType.List) { { BlockIds = new List <long>(); TList _list13 = iprot.ReadListBegin(); for (int _i14 = 0; _i14 < _list13.Count; ++_i14) { long _elem15; _elem15 = iprot.ReadI64(); BlockIds.Add(_elem15); } iprot.ReadListEnd(); } } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 15: if (field.Type == TType.I32) { InMemoryPercentage = iprot.ReadI32(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 16: if (field.Type == TType.I64) { LastModificationTimeMs = iprot.ReadI64(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 17: if (field.Type == TType.I64) { Ttl = iprot.ReadI64(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 18: if (field.Type == TType.String) { Owner = iprot.ReadString(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 19: if (field.Type == TType.String) { Group = iprot.ReadString(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 20: if (field.Type == TType.I32) { Mode = iprot.ReadI32(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 21: if (field.Type == TType.String) { PersistenceState = iprot.ReadString(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 22: if (field.Type == TType.Bool) { MountPoint = iprot.ReadBool(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 23: if (field.Type == TType.List) { { FileBlockInfos = new List <FileBlockInfo>(); TList _list16 = iprot.ReadListBegin(); for (int _i17 = 0; _i17 < _list16.Count; ++_i17) { FileBlockInfo _elem18; _elem18 = new FileBlockInfo(); _elem18.Read(iprot); FileBlockInfos.Add(_elem18); } iprot.ReadListEnd(); } } else { TProtocolUtil.Skip(iprot, field.Type); } break; default: TProtocolUtil.Skip(iprot, field.Type); break; } iprot.ReadFieldEnd(); } iprot.ReadStructEnd(); } finally { iprot.DecrementRecursionDepth(); } }
/// <summary> /// 检查是否处于断点续传状态 /// </summary> /// <param name="responsed"></param> /// <param name="request"></param> /// <param name="info"></param> /// <param name="fsPosition"></param> /// <param name="positionOffset"></param> /// <param name="sendBlockSize"></param> private void OfflineReSendCheck(FileTransferResponsed responsed, FileTransferRequest request, FileBlockInfo info, long fsPosition, ref long positionOffset, ref int sendBlockSize) { if (responsed.RemoteStreamPosition == fsPosition) { request.BlockIndex++; } else { int rBlockIndex = 0; long rPosition = 0; ClacBlockIndex(info, responsed.RemoteStreamPosition, out rBlockIndex, out rPosition); request.BlockIndex = rBlockIndex; positionOffset = rPosition; } if (info.BlockSize > info.FileLength - positionOffset) { sendBlockSize = (int)(info.FileLength - positionOffset); } }
/// <summary> /// 根据反馈,重发文件 /// </summary> /// <param name="path"></param> /// <param name="fileBlockInfo"></param> /// <param name="request"></param> /// <param name="checkResponsed"></param> private bool ReSendFile(string path, FileBlockInfo fileBlockInfo, FileTransferRequest request, FileTransferResponsed checkResponsed) { return(ReSendFile(path, _fileStreamDic[path], fileBlockInfo, request, checkResponsed)); }
/// <summary> /// 传输文件 /// </summary> /// <param name="path"></param> /// <param name="progress"></param> /// <param name="fileBlockInfo"></param> /// <param name="request"></param> private void OnSendFile(string path, ProgressMessage progress, FileBlockInfo fileBlockInfo, FileTransferRequest request) { var fs = _fileStreamDic[path]; OnSendFile(path, fs, progress, fileBlockInfo, request); }
private void OnSendFile(string path, FileStream fs, ProgressMessage progress, FileBlockInfo fileBlockInfo, FileTransferRequest request) { fs.Position = 0; long positionOffset = 0; int sendBlockSize = request.EachBlockSize; if (progress != null) { progress.Title = fileBlockInfo.FileName; progress.MaxValue = fileBlockInfo.BlockCount; progress.StateMsg = "Prepare OK"; try { _progressHandler?.OnSendStart(fileBlockInfo.FileName, progress); #if DEBUG Console.WriteLine(progress.StateMsg); #endif } catch { // ignored } } //普通循环读取和发送文件 while (fs.Position < fs.Length) { request.BlockData = null; request.SeekOffset = positionOffset; var buffer = ReadFileBytes(path, fs.Position, sendBlockSize); request.BlockData = buffer; var responsed = _sendAdapter.UpdateFileData(request); _responsedHandler?.HandleResponsed(responsed);//等待外部对回应进行处理 positionOffset = responsed.RemoteStreamPosition; OfflineReSendCheck(responsed, request, fileBlockInfo, fs.Position, ref positionOffset, ref sendBlockSize); //文件离线重传检查 fs.Position = positionOffset; //进度对齐 if (progress != null) { progress.ProgressValue = request.BlockIndex; try { _progressHandler?.OnSending(fileBlockInfo.FileName); #if DEBUG Console.Write($"{progress.ProgressValue}/{progress.MaxValue}"); Console.SetCursorPosition(0, Console.CursorTop); ClearCurrentConsoleLine(); #endif } catch { // ignored } } } }