/// <inheritdoc />
public Task <bool> VerifyChecksumAsync(string fileId, string algorithm, byte[] checksum, CancellationToken _)
{
var valid = false;
var internalFileId = new InternalFileId(fileId);
using (var dataStream = _fileRepFactory.Data(internalFileId, _config).GetStream(FileMode.Open, FileAccess.ReadWrite, FileShare.Read))
{
var chunkPositionFile = _fileRepFactory.ChunkStartPosition(internalFileId, _config);
var chunkStartPosition = chunkPositionFile.ReadFirstLineAsLong(true, 0);
var chunkCompleteFile = _fileRepFactory.ChunkComplete(internalFileId, _config);
// Only verify the checksum if the entire lastest chunk has been written.
// If not, just discard the last chunk as it won't match the checksum anyway.
if (chunkCompleteFile.Exist())
{
var calculateSha1 = dataStream.CalculateSha1(chunkStartPosition);
valid = checksum.SequenceEqual(calculateSha1);
}
if (!valid)
{
dataStream.Seek(0, SeekOrigin.Begin);
dataStream.SetLength(chunkStartPosition);
}
}
return(Task.FromResult(valid));
}
private InternalFileRep InitializeChunk(InternalFileId internalFileId, long totalDiskFileLength)
{
var chunkComplete = _fileRepFactory.ChunkComplete(internalFileId, _config);
chunkComplete.Delete();
_fileRepFactory.ChunkStartPosition(internalFileId, _config).Write(totalDiskFileLength.ToString());
return(chunkComplete);
}
/// <inheritdoc />
public Task <ITusFile> GetFileAsync(string fileId, CancellationToken _)
{
var internalFileId = new InternalFileId(fileId);
var data = _fileRepFactory.Data(internalFileId, _config);
return(Task.FromResult <ITusFile>(data.Exist()
? new TusDiskFile(data, _fileRepFactory.Metadata(internalFileId, _config))
: null));
}
private async Task FlushFileToDisk(InternalFileId internalFileId, byte[] fileWriteBuffer, Stream stream, int writeBufferNextFreeIndex)
{
var path = _fileRepFactory.Data(internalFileId, _config).Path;
var client = new FtpClient(_config.Value.ServerUri, _config.Value.ServerPort, _config.Value.ServerUsername, _config.Value.ServerPassword);
await stream.WriteAsync(fileWriteBuffer, 0, writeBufferNextFreeIndex);
await stream.FlushAsync();
}
private InternalFileRep Create(InternalFileId fileId, string extension, IOptions <FtpServerSettings> config)
{
var fileName = fileId.FileId;
if (!string.IsNullOrEmpty(extension))
{
fileName += "." + extension;
}
return(new InternalFileRep(fileId.FileId, System.IO.Path.Combine(_directoryPath, fileName), config));
}
/// <inheritdoc />
public Task DeleteFileAsync(string fileId, CancellationToken _)
{
var internalFileId = new InternalFileId(fileId);
return(Task.Run(() =>
{
_fileRepFactory.Data(internalFileId, _config).Delete();
_fileRepFactory.UploadLength(internalFileId, _config).Delete();
_fileRepFactory.Metadata(internalFileId, _config).Delete();
_fileRepFactory.UploadConcat(internalFileId, _config).Delete();
_fileRepFactory.ChunkStartPosition(internalFileId, _config).Delete();
_fileRepFactory.ChunkComplete(internalFileId, _config).Delete();
_fileRepFactory.Expiration(internalFileId, _config).Delete();
}));
}
/// <inheritdoc />
public async Task <string> CreateFileAsync(long uploadLength, string metadata, CancellationToken cancellationToken)
{
var fileId = new InternalFileId();
var client = new FtpClient(_config.Value.ServerUri, _config.Value.ServerPort, _config.Value.ServerUsername, _config.Value.ServerPassword);
client.OpenWrite(_fileRepFactory.Data(fileId, _config).Path).Dispose();
if (uploadLength != -1)
{
await SetUploadLengthAsync(fileId.FileId, uploadLength, cancellationToken);
}
_fileRepFactory.Metadata(fileId, _config).Write(metadata);
return(fileId.FileId);
}
/// <inheritdoc />
public async Task <string> CreateFinalFileAsync(string[] partialFiles, string metadata, CancellationToken cancellationToken)
{
var partialInternalFileReps = partialFiles.Select(f =>
{
var partialData = _fileRepFactory.Data(new InternalFileId(f), _config);
if (!partialData.Exist())
{
throw new TusStoreException($"File {f} does not exist");
}
return(partialData);
}).ToArray();
var length = partialInternalFileReps.Sum(f => f.GetLength());
var fileId = await CreateFileAsync(length, metadata, cancellationToken);
var internalFileId = new InternalFileId(fileId);
_fileRepFactory.UploadConcat(internalFileId, _config).Write(new FileConcatFinal(partialFiles).GetHeader());
using (var finalFile = _fileRepFactory.Data(internalFileId, _config).GetStream(FileMode.Open, FileAccess.Write, FileShare.None))
{
foreach (var partialFile in partialInternalFileReps)
{
using var partialStream = partialFile.GetStream(FileMode.Open, FileAccess.Read, FileShare.Read);
await partialStream.CopyToAsync(finalFile);
}
}
if (_deletePartialFilesOnConcat)
{
await Task.WhenAll(partialInternalFileReps.Select(f => DeleteFileAsync(f.FileId, cancellationToken)));
}
return(fileId);
}
/// <inheritdoc />
public async Task <long> AppendDataAsync(string fileId, Stream stream, CancellationToken cancellationToken)
{
var client = new FtpClient(_config.Value.ServerUri, _config.Value.ServerPort, _config.Value.ServerUsername, _config.Value.ServerPassword);
var internalFileId = new InternalFileId(fileId);
var httpReadBuffer = _bufferPool.Rent(_maxReadBufferSize);
var fileWriteBuffer = _bufferPool.Rent(Math.Max(_maxWriteBufferSize, _maxReadBufferSize));
try
{
var fileUploadLengthProvidedDuringCreate = await GetUploadLengthAsync(fileId, cancellationToken);
using var diskFileStream = await client.OpenAppendAsync(_fileRepFactory.Data (internalFileId, _config).Path);
var totalDiskFileLength = diskFileStream.Length;
if (fileUploadLengthProvidedDuringCreate == totalDiskFileLength)
{
return(0);
}
var chunkCompleteFile = InitializeChunk(internalFileId, totalDiskFileLength);
int numberOfbytesReadFromClient;
var bytesWrittenThisRequest = 0L;
var clientDisconnectedDuringRead = false;
var writeBufferNextFreeIndex = 0;
do
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
numberOfbytesReadFromClient = await stream.ReadAsync(httpReadBuffer, 0, _maxReadBufferSize, cancellationToken);
clientDisconnectedDuringRead = cancellationToken.IsCancellationRequested;
totalDiskFileLength += numberOfbytesReadFromClient;
if (totalDiskFileLength > fileUploadLengthProvidedDuringCreate)
{
throw new TusStoreException($"Stream contains more data than the file's upload length. Stream data: {totalDiskFileLength}, upload length: {fileUploadLengthProvidedDuringCreate}.");
}
// Can we fit the read data into the write buffer? If not flush it now.
if (writeBufferNextFreeIndex + numberOfbytesReadFromClient > _maxWriteBufferSize)
{
await FlushFileToDisk(internalFileId, fileWriteBuffer, diskFileStream, writeBufferNextFreeIndex);
writeBufferNextFreeIndex = 0;
}
Array.Copy(
sourceArray: httpReadBuffer,
sourceIndex: 0,
destinationArray: fileWriteBuffer,
destinationIndex: writeBufferNextFreeIndex,
length: numberOfbytesReadFromClient);
writeBufferNextFreeIndex += numberOfbytesReadFromClient;
bytesWrittenThisRequest += numberOfbytesReadFromClient;
} while (numberOfbytesReadFromClient != 0);
// Flush the remaining buffer to disk.
if (writeBufferNextFreeIndex != 0)
{
await FlushFileToDisk(internalFileId, fileWriteBuffer, diskFileStream, writeBufferNextFreeIndex);
}
if (!clientDisconnectedDuringRead)
{
MarkChunkComplete(chunkCompleteFile);
}
return(bytesWrittenThisRequest);
}
finally
{
_bufferPool.Return(httpReadBuffer);
_bufferPool.Return(fileWriteBuffer);
}
}
public InternalFileRep ChunkComplete(InternalFileId fileId, IOptions <FtpServerSettings> config) => Create(fileId, "chunkcomplete", config);
public InternalFileRep ChunkStartPosition(InternalFileId fileId, IOptions <FtpServerSettings> config) => Create(fileId, "chunkstart", config);
public InternalFileRep Expiration(InternalFileId fileId, IOptions <FtpServerSettings> config) => Create(fileId, "expiration", config);
public InternalFileRep Metadata(InternalFileId fileId, IOptions <FtpServerSettings> config) => Create(fileId, "metadata", config);
public InternalFileRep UploadConcat(InternalFileId fileId, IOptions <FtpServerSettings> config) => Create(fileId, "uploadconcat", config);
public InternalFileRep UploadLength(InternalFileId fileId, IOptions <FtpServerSettings> config) => Create(fileId, "uploadlength", config);