public IActionResult Add([FromRoute] long fileId, [FromBody] FileBlockInfo fileBlockInfo)
{
FileBlock fileBlock = _db.FileBlocks.FirstOrDefault(fb => fb.FileId == fileId && fb.Number == fileBlockInfo.Number);
using (MD5 md5Hash = MD5.Create())
{
byte[] rawContent = Convert.FromBase64String(fileBlockInfo.Value);
if (fileBlock == null) //Create new fileBlock
{
_db.FileBlocks.Add(new FileBlock()
{
FileId = fileId,
Number = fileBlockInfo.Number,
Content = rawContent,
Checksum = MD5Utils.GetMd5Hash(md5Hash, rawContent.ToString())
});
}
else //Update existing fileBlock
{
fileBlock.Content = rawContent;
fileBlock.Checksum = MD5Utils.GetMd5Hash(md5Hash, rawContent.ToString());
_db.FileBlocks.Update(fileBlock);
}
_db.SaveChanges();
}
return(StatusCode((int)HttpStatusCode.OK));
}
public FileBlockInfo ClacFileBlockInfo(string path, FileStream fs, int blockSize = 8192)
{
long fileLength = fs.Length;
if (fileLength <= 0)
{
throw new ArgumentOutOfRangeException("文件长度不应为0");
}
long position = fs.Position;
long blockCountF = fileLength / blockSize;
int lastBlockSize = (int)(fileLength - blockCountF * blockSize);
var blockCount = lastBlockSize <= 0 ? (int)blockCountF : (int)blockCountF + 1;
if (fileLength < blockSize)
{
blockCount = 1;
lastBlockSize = (int)fileLength;
blockSize = lastBlockSize;
}
var info = new FileBlockInfo()
{
BlockCount = blockCount,
BlockSize = blockSize,
LastBlockSize = lastBlockSize,
FileName = Path.GetFileName(path),
FileMd5 = Md5.GetMd5WithFileStream(fs, fs.Position),
FileLength = fileLength
};
fs.Position = position;
return(info);
}
private FileTransferRequest GenerateTransferRequest(FileBlockInfo info)
{
int seed = GetRandomSeed();
var random = new Random(seed);
return(GenerateTransferRequest(info, random.Next(int.MinValue, int.MaxValue)));
}
/// <summary>
/// 计算每个文件块的MD5值
/// </summary>
/// <param name="info"></param>
/// <param name="fs"></param>
/// <returns></returns>
private List <BlockTransferRequest> ClacFileEachBlockMd5(FileBlockInfo info, FileStream fs)
{
var position = fs.Position;
var list = new List <BlockTransferRequest>();
int sendBlockSize = info.BlockSize;
int blockIndex = 0;
while (fs.Position < fs.Length)
{
var r = new BlockTransferRequest()
{
FileName = info.FileName,
BlockIndex = blockIndex,
BlockSize = sendBlockSize,
SeekOffset = (int)fs.Position
};
byte[] buffer = new byte[sendBlockSize];
fs.Read(buffer, 0, buffer.Length);//read之后，Position会变,所以要先生成对象
r.BlockMd5 = Md5.GetMd5WithBytes(buffer);
list.Add(r);
blockIndex++;//增加索引序号,别漏了
if (sendBlockSize > fs.Length - fs.Position)
{
sendBlockSize = (int)(fs.Length - fs.Position);
}
}
fs.Position = position;
return(list);
}
public async Task <IActionResult> UploadFile()
{
foreach (var file in Request.Form.Files)
{
if (file.Length == 0)
{
continue;
}
using (var contentStream = file.OpenReadStream())
{
var fileBlockInfo = new FileBlockInfo
{
FileId = Request.Form["flowIdentifier"],
BlockId = Request.Form["flowIdentifier"],
FileName = Request.Form["flowFilename"],
FileSize = long.Parse(Request.Form["flowTotalSize"]),
BlockSize = long.Parse(Request.Form["flowChunkSize"]),
SequenceNum = int.Parse(Request.Form["flowChunkNumber"]),
TotalBlocksCount = int.Parse(Request.Form["flowTotalChunks"])
};
var uploadToAzure = Convert.ToBoolean(Request.Form["toAzure"]);
await _fileUploadManager.AddFileBlockAsync(_serviceProvider, fileBlockInfo, contentStream, uploadToAzure);
}
}
return(Ok());
}
public bool SetFileBlockContent(long fileId, FileBlockInfo fileBlockInfo)
{
string route = $"{BaseAddress}/File/{fileId}/FileBlock";
var result = QueryRequest(route, HttpMethod.Post,
new StringContent(JsonConvert.SerializeObject(fileBlockInfo), Encoding.UTF8, "application/json"));
if (result.status != HttpStatusCode.OK)
{
return(false);
}
return(true);
}
public FileTransferRequest GenerateTransferRequest(FileBlockInfo info, int randomSeed)
{
var request = new FileTransferRequest()
{
BlockIndex = 0,
BlockCount = info.BlockCount,
LastBlockSize = info.LastBlockSize,
EachBlockSize = info.BlockSize,
FileMd5 = info.FileMd5,
FileName = info.FileName,
FileSuffix = Path.GetExtension(info.FileName),
FileSize = info.FileLength,
IsSendingOver = false,
SeekOffset = 0,
BlockData = null,
RequestId = randomSeed
};
return(request);
}
/// <summary>
/// 计算文件块索引
/// </summary>
/// <param name="info"></param>
/// <param name="remotePosition"></param>
/// <param name="realBlockIndex"></param>
/// <param name="realFsPosition"></param>
private void ClacBlockIndex(FileBlockInfo info, long remotePosition, out int realBlockIndex, out long realFsPosition)
{
var realIndex = (int)(remotePosition / info.BlockSize);
var realPosition = info.BlockSize * realIndex;
if (realPosition + info.LastBlockSize == info.FileLength)//判断是否为倒数第二个Block
{
realBlockIndex = info.BlockCount - 1;
realFsPosition = info.FileLength - info.LastBlockSize;
}
else if (remotePosition == info.FileLength)//判断是否为最后一个Block
{
realBlockIndex = info.BlockCount;
realFsPosition = info.FileLength;
}
else
{
realBlockIndex = realPosition;
realFsPosition = realIndex;
}
}
/// <summary>
/// 根据反馈结果，重发文件
/// </summary>
/// <param name="path"></param>
/// <param name="fs"></param>
/// <param name="fileBlockInfo"></param>
/// <param name="request"></param>
/// <param name="checkResponsed"></param>
public bool ReSendFile(string path, FileStream fs, FileBlockInfo fileBlockInfo, FileTransferRequest request, FileTransferResponsed checkResponsed)
{
int resendCountOut = 0;
while (!checkResponsed.FileMd5CheckResult && resendCountOut < 3)//最多重发三次
{
#if DEBUG
Console.WriteLine("ReSending File, count " + resendCountOut);
#endif
//MD5检查失败重发
var blockMsgList = ClacFileEachBlockMd5(fileBlockInfo, fs);
if (blockMsgList.Count != fileBlockInfo.BlockCount)
{
throw new Exception("校准MD5时，Block数量计算错误!");
}
int size = request.EachBlockSize;
request.IsSendingOver = false;
for (int i = 0; i < blockMsgList.Count; i++)
{
var blockCheck = _sendAdapter.UpdateFileBlockMessage(blockMsgList[i]);
if (blockCheck.IsError)
{
if (i == blockMsgList.Count - 1)
{
size = request.LastBlockSize;
}
request.BlockIndex = i;
request.SeekOffset = i * fileBlockInfo.BlockSize;
request.BlockData = ReadFileBytes(path, request.SeekOffset, size);
_sendAdapter.UpdateFileData(request);
}
}
request.IsSendingOver = true;
checkResponsed = _sendAdapter.UpdateFileData(request);
resendCountOut++;
}
return(checkResponsed.FileMd5CheckResult);
}
public void Read(TProtocol iprot)
{
iprot.IncrementRecursionDepth();
try
{
TField field;
iprot.ReadStructBegin();
while (true)
{
field = iprot.ReadFieldBegin();
if (field.Type == TType.Stop)
{
break;
}
switch (field.ID)
{
case 1:
if (field.Type == TType.I64)
{
FileId = iprot.ReadI64();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 2:
if (field.Type == TType.String)
{
Name = iprot.ReadString();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 3:
if (field.Type == TType.String)
{
Path = iprot.ReadString();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 4:
if (field.Type == TType.String)
{
UfsPath = iprot.ReadString();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 5:
if (field.Type == TType.I64)
{
Length = iprot.ReadI64();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 6:
if (field.Type == TType.I64)
{
BlockSizeBytes = iprot.ReadI64();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 7:
if (field.Type == TType.I64)
{
CreationTimeMs = iprot.ReadI64();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 8:
if (field.Type == TType.Bool)
{
Completed = iprot.ReadBool();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 9:
if (field.Type == TType.Bool)
{
Folder = iprot.ReadBool();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 10:
if (field.Type == TType.Bool)
{
Pinned = iprot.ReadBool();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 11:
if (field.Type == TType.Bool)
{
Cacheable = iprot.ReadBool();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 12:
if (field.Type == TType.Bool)
{
Persisted = iprot.ReadBool();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 13:
if (field.Type == TType.List)
{
{
BlockIds = new List <long>();
TList _list13 = iprot.ReadListBegin();
for (int _i14 = 0; _i14 < _list13.Count; ++_i14)
{
long _elem15;
_elem15 = iprot.ReadI64();
BlockIds.Add(_elem15);
}
iprot.ReadListEnd();
}
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 15:
if (field.Type == TType.I32)
{
InMemoryPercentage = iprot.ReadI32();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 16:
if (field.Type == TType.I64)
{
LastModificationTimeMs = iprot.ReadI64();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 17:
if (field.Type == TType.I64)
{
Ttl = iprot.ReadI64();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 18:
if (field.Type == TType.String)
{
Owner = iprot.ReadString();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 19:
if (field.Type == TType.String)
{
Group = iprot.ReadString();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 20:
if (field.Type == TType.I32)
{
Mode = iprot.ReadI32();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 21:
if (field.Type == TType.String)
{
PersistenceState = iprot.ReadString();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 22:
if (field.Type == TType.Bool)
{
MountPoint = iprot.ReadBool();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 23:
if (field.Type == TType.List)
{
{
FileBlockInfos = new List <FileBlockInfo>();
TList _list16 = iprot.ReadListBegin();
for (int _i17 = 0; _i17 < _list16.Count; ++_i17)
{
FileBlockInfo _elem18;
_elem18 = new FileBlockInfo();
_elem18.Read(iprot);
FileBlockInfos.Add(_elem18);
}
iprot.ReadListEnd();
}
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
default:
TProtocolUtil.Skip(iprot, field.Type);
break;
}
iprot.ReadFieldEnd();
}
iprot.ReadStructEnd();
}
finally
{
iprot.DecrementRecursionDepth();
}
}
/// <summary>
/// 检查是否处于断点续传状态
/// </summary>
/// <param name="responsed"></param>
/// <param name="request"></param>
/// <param name="info"></param>
/// <param name="fsPosition"></param>
/// <param name="positionOffset"></param>
/// <param name="sendBlockSize"></param>
private void OfflineReSendCheck(FileTransferResponsed responsed, FileTransferRequest request, FileBlockInfo info, long fsPosition, ref long positionOffset, ref int sendBlockSize)
{
if (responsed.RemoteStreamPosition == fsPosition)
{
request.BlockIndex++;
}
else
{
int rBlockIndex = 0;
long rPosition = 0;
ClacBlockIndex(info, responsed.RemoteStreamPosition, out rBlockIndex, out rPosition);
request.BlockIndex = rBlockIndex;
positionOffset = rPosition;
}
if (info.BlockSize > info.FileLength - positionOffset)
{
sendBlockSize = (int)(info.FileLength - positionOffset);
}
}
/// <summary>
/// 根据反馈，重发文件
/// </summary>
/// <param name="path"></param>
/// <param name="fileBlockInfo"></param>
/// <param name="request"></param>
/// <param name="checkResponsed"></param>
private bool ReSendFile(string path, FileBlockInfo fileBlockInfo, FileTransferRequest request, FileTransferResponsed checkResponsed)
{
return(ReSendFile(path, _fileStreamDic[path], fileBlockInfo, request, checkResponsed));
}
/// <summary>
/// 传输文件
/// </summary>
/// <param name="path"></param>
/// <param name="progress"></param>
/// <param name="fileBlockInfo"></param>
/// <param name="request"></param>
private void OnSendFile(string path, ProgressMessage progress, FileBlockInfo fileBlockInfo, FileTransferRequest request)
{
var fs = _fileStreamDic[path];
OnSendFile(path, fs, progress, fileBlockInfo, request);
}
private void OnSendFile(string path, FileStream fs, ProgressMessage progress, FileBlockInfo fileBlockInfo, FileTransferRequest request)
{
fs.Position = 0;
long positionOffset = 0;
int sendBlockSize = request.EachBlockSize;
if (progress != null)
{
progress.Title = fileBlockInfo.FileName;
progress.MaxValue = fileBlockInfo.BlockCount;
progress.StateMsg = "Prepare OK";
try
{
_progressHandler?.OnSendStart(fileBlockInfo.FileName, progress);
#if DEBUG
Console.WriteLine(progress.StateMsg);
#endif
}
catch
{
// ignored
}
}
//普通循环读取和发送文件
while (fs.Position < fs.Length)
{
request.BlockData = null;
request.SeekOffset = positionOffset;
var buffer = ReadFileBytes(path, fs.Position, sendBlockSize);
request.BlockData = buffer;
var responsed = _sendAdapter.UpdateFileData(request);
_responsedHandler?.HandleResponsed(responsed);//等待外部对回应进行处理
positionOffset = responsed.RemoteStreamPosition;
OfflineReSendCheck(responsed, request, fileBlockInfo, fs.Position, ref positionOffset, ref sendBlockSize); //文件离线重传检查
fs.Position = positionOffset; //进度对齐
if (progress != null)
{
progress.ProgressValue = request.BlockIndex;
try
{
_progressHandler?.OnSending(fileBlockInfo.FileName);
#if DEBUG
Console.Write($"{progress.ProgressValue}/{progress.MaxValue}");
Console.SetCursorPosition(0, Console.CursorTop);
ClearCurrentConsoleLine();
#endif
}
catch
{
// ignored
}
}
}
}
