/// <inheritdoc />
public async Task <string> CreatePartialFileAsync(long uploadLength, string metadata, CancellationToken cancellationToken)
{
var fileId = await CreateFileAsync(uploadLength, metadata, cancellationToken);
_fileRepFactory.UploadConcat(await InternalFileId.Parse(_fileIdProvider, fileId)).Write(new FileConcatPartial().GetHeader());
return(fileId);
}
/// <inheritdoc />
public async Task <bool> VerifyChecksumAsync(string fileId, string algorithm, byte[] checksum, CancellationToken _)
{
var valid = false;
var internalFileId = await InternalFileId.Parse(_fileIdProvider, fileId);
using (var dataStream = _fileRepFactory.Data(internalFileId).GetStream(FileMode.Open, FileAccess.ReadWrite, FileShare.Read))
{
var chunkPositionFile = _fileRepFactory.ChunkStartPosition(internalFileId);
var chunkStartPosition = chunkPositionFile.ReadFirstLineAsLong(true, 0);
var chunkCompleteFile = _fileRepFactory.ChunkComplete(internalFileId);
// Only verify the checksum if the entire lastest chunk has been written.
// If not, just discard the last chunk as it won't match the checksum anyway.
if (chunkCompleteFile.Exist())
{
var calculateSha1 = dataStream.CalculateSha1(chunkStartPosition);
valid = checksum.SequenceEqual(calculateSha1);
}
if (!valid)
{
dataStream.Seek(0, SeekOrigin.Begin);
dataStream.SetLength(chunkStartPosition);
}
}
return(valid);
}
/// <inheritdoc />
public async Task <long?> GetUploadLengthAsync(string fileId, CancellationToken _)
{
var firstLine = _fileRepFactory.UploadLength(await InternalFileId.Parse(_fileIdProvider, fileId)).ReadFirstLine(true);
return(firstLine == null
? (long?)null
: long.Parse(firstLine));
}
/// <inheritdoc />
public async Task <FileConcat> GetUploadConcatAsync(string fileId, CancellationToken _)
{
var firstLine = _fileRepFactory.UploadConcat(await InternalFileId.Parse(_fileIdProvider, fileId)).ReadFirstLine(true);
return(string.IsNullOrWhiteSpace(firstLine)
? null
: new UploadConcat(firstLine).Type);
}
/// <inheritdoc />
public async Task <DateTimeOffset?> GetExpirationAsync(string fileId, CancellationToken _)
{
var expiration = _fileRepFactory.Expiration(await InternalFileId.Parse(_fileIdProvider, fileId)).ReadFirstLine(true);
return(expiration == null
? (DateTimeOffset?)null
: DateTimeOffset.ParseExact(expiration, "O", null));
}
private InternalFileRep InitializeChunk(InternalFileId internalFileId, long totalDiskFileLength)
{
var chunkComplete = _fileRepFactory.ChunkComplete(internalFileId);
chunkComplete.Delete();
_fileRepFactory.ChunkStartPosition(internalFileId).Write(totalDiskFileLength.ToString());
return(chunkComplete);
}
/// <inheritdoc />
public Task <ITusFile> GetFileAsync(string fileId, CancellationToken cancellationToken)
{
var internalFileId = new InternalFileId(fileId);
var data = _fileRepFactory.Data(internalFileId);
return(Task.FromResult <ITusFile>(data.Exist()
? new TusDiskFile(data, _fileRepFactory.Metadata(internalFileId))
: null));
}
/// <inheritdoc />
public async Task <long> AppendDataAsync(string fileId, Stream stream, CancellationToken cancellationToken)
{
var internalFileId = new InternalFileId(fileId);
long bytesWritten = 0;
var uploadLength = await GetUploadLengthAsync(fileId, cancellationToken);
using (var file = _fileRepFactory.Data(internalFileId).GetStream(FileMode.Append, FileAccess.Write, FileShare.None))
{
var fileLength = file.Length;
if (uploadLength == fileLength)
{
return(0);
}
var chunkStart = _fileRepFactory.ChunkStartPosition(internalFileId);
var chunkComplete = _fileRepFactory.ChunkComplete(internalFileId);
if (chunkComplete.Exist())
{
chunkStart.Delete();
chunkComplete.Delete();
}
if (!chunkStart.Exist())
{
chunkStart.Write(fileLength.ToString());
}
int bytesRead;
do
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
var buffer = new byte[ByteChunkSize];
bytesRead = await stream.ReadAsync(buffer, 0, ByteChunkSize, cancellationToken);
fileLength += bytesRead;
if (fileLength > uploadLength)
{
throw new TusStoreException(
$"Stream contains more data than the file's upload length. Stream data: {fileLength}, upload length: {uploadLength}.");
}
file.Write(buffer, 0, bytesRead);
bytesWritten += bytesRead;
} while (bytesRead != 0);
// Chunk is complete. Mark it as complete.
chunkComplete.Write("1");
return(bytesWritten);
}
}
/// <inheritdoc />
public async Task <ITusFile> GetFileAsync(string fileId, CancellationToken _)
{
var internalFileId = await InternalFileId.Parse(_fileIdProvider, fileId);
var data = _fileRepFactory.Data(internalFileId);
return(data.Exist()
? new TusDiskFile(data, _fileRepFactory.Metadata(internalFileId))
: null);
}
private InternalFileRep Create(InternalFileId fileId, string extension)
{
string fileName = fileId;
if (!string.IsNullOrEmpty(extension))
{
fileName += "." + extension;
}
return(new InternalFileRep(fileId, System.IO.Path.Combine(_directoryPath, fileName)));
}
/// <inheritdoc />
public async Task <string> CreateFileAsync(long uploadLength, string metadata, CancellationToken cancellationToken)
{
var fileId = await InternalFileId.CreateNew(_fileIdProvider, metadata);
new FileStream(_fileRepFactory.Data(fileId).Path, FileMode.CreateNew).Dispose();
if (uploadLength != -1)
{
await SetUploadLengthAsync(fileId, uploadLength, cancellationToken);
}
_fileRepFactory.Metadata(fileId).Write(metadata);
return(fileId);
}
/// <inheritdoc />
public async Task <string> CreateFileAsync(long uploadLength, string metadata, CancellationToken cancellationToken)
{
var fileId = new InternalFileId();
File.Create(_fileRepFactory.Data(fileId).Path).Dispose();
if (uploadLength != -1)
{
await SetUploadLengthAsync(fileId.FileId, uploadLength, cancellationToken);
}
_fileRepFactory.Metadata(fileId).Write(metadata);
return(fileId.FileId);
}
/// <inheritdoc />
public async Task DeleteFileAsync(string fileId, CancellationToken _)
{
var internalFileId = await InternalFileId.Parse(_fileIdProvider, fileId);
await Task.Run(() =>
{
_fileRepFactory.Data(internalFileId).Delete();
_fileRepFactory.UploadLength(internalFileId).Delete();
_fileRepFactory.Metadata(internalFileId).Delete();
_fileRepFactory.UploadConcat(internalFileId).Delete();
_fileRepFactory.ChunkStartPosition(internalFileId).Delete();
_fileRepFactory.ChunkComplete(internalFileId).Delete();
_fileRepFactory.Expiration(internalFileId).Delete();
});
}
/// <inheritdoc />
public Task DeleteFileAsync(string fileId, CancellationToken cancellationToken)
{
var internalFileId = new InternalFileId(fileId);
return(Task.Run(() =>
{
_fileRepFactory.Data(internalFileId).Delete();
_fileRepFactory.UploadLength(internalFileId).Delete();
_fileRepFactory.Metadata(internalFileId).Delete();
_fileRepFactory.UploadConcat(internalFileId).Delete();
_fileRepFactory.Expiration(internalFileId).Delete();
_fileRepFactory.ChunkStartPosition(internalFileId).Delete();
_fileRepFactory.ChunkComplete(internalFileId).Delete();
}, cancellationToken));
}
/// <inheritdoc />
public async Task <IEnumerable <string> > GetExpiredFilesAsync(CancellationToken _)
{
List <string> expiredFiles = new List <string>();
foreach (var file in Directory.EnumerateFiles(_directoryPath, "*.expiration"))
{
var f = await InternalFileId.Parse(_fileIdProvider, Path.GetFileNameWithoutExtension(file));
if (FileHasExpired(f) && FileIsIncomplete(f))
{
expiredFiles.Add(f);
}
}
return(expiredFiles);
bool FileHasExpired(InternalFileId fileId)
{
var firstLine = _fileRepFactory.Expiration(fileId).ReadFirstLine();
return(!string.IsNullOrWhiteSpace(firstLine) &&
DateTimeOffset.ParseExact(firstLine, "O", null).HasPassed());
}
bool FileIsIncomplete(InternalFileId fileId)
{
var uploadLength = _fileRepFactory.UploadLength(fileId).ReadFirstLineAsLong(fileIsOptional: true, defaultValue: long.MinValue);
if (uploadLength == long.MinValue)
{
return(true);
}
var dataFile = _fileRepFactory.Data(fileId);
if (!dataFile.Exist())
{
return(true);
}
return(uploadLength != dataFile.GetLength());
}
}
/// <inheritdoc />
public async Task <string> CreateFinalFileAsync(string[] partialFiles, string metadata, CancellationToken cancellationToken)
{
var partialInternalFileReps = partialFiles.Select(f =>
{
var partialData = _fileRepFactory.Data(new InternalFileId(f));
if (!partialData.Exist())
{
throw new TusStoreException($"File {f} does not exist");
}
return(partialData);
}).ToArray();
var length = partialInternalFileReps.Sum(f => f.GetLength());
var fileId = await CreateFileAsync(length, metadata, cancellationToken);
var internalFileId = new InternalFileId(fileId);
_fileRepFactory.UploadConcat(internalFileId).Write(new FileConcatFinal(partialFiles).GetHeader());
using (var finalFile = _fileRepFactory.Data(internalFileId).GetStream(FileMode.Open, FileAccess.Write, FileShare.None))
{
foreach (var partialFile in partialInternalFileReps)
{
using (var partialStream = partialFile.GetStream(FileMode.Open, FileAccess.Read, FileShare.Read))
{
partialStream.CopyTo(finalFile);
}
}
}
// ReSharper disable once InvertIf
if (_deletePartialFilesOnConcat)
{
await Task.WhenAll(partialInternalFileReps.Select(f => DeleteFileAsync(f.FileId, cancellationToken)));
}
return(fileId);
}
/// <inheritdoc />
public async Task <string> CreateFinalFileAsync(string[] partialFiles, string metadata, CancellationToken cancellationToken)
{
InternalFileRep[] partialInternalFileReps = new InternalFileRep[partialFiles.Length];
for (int i = 0; i < partialFiles.Length; i++)
{
partialInternalFileReps[i] = _fileRepFactory.Data(await InternalFileId.Parse(_fileIdProvider, partialFiles[i]));
if (!partialInternalFileReps[i].Exist())
{
throw new TusStoreException($"File {partialFiles[i]} does not exist");
}
}
var length = partialInternalFileReps.Sum(f => f.GetLength());
var fileId = await CreateFileAsync(length, metadata, cancellationToken);
var internalFileId = await InternalFileId.Parse(_fileIdProvider, fileId);
_fileRepFactory.UploadConcat(internalFileId).Write(new FileConcatFinal(partialFiles).GetHeader());
using (var finalFile = _fileRepFactory.Data(internalFileId).GetStream(FileMode.Open, FileAccess.Write, FileShare.None))
{
foreach (var partialFile in partialInternalFileReps)
{
using var partialStream = partialFile.GetStream(FileMode.Open, FileAccess.Read, FileShare.Read);
await partialStream.CopyToAsync(finalFile);
}
}
if (_deletePartialFilesOnConcat)
{
await Task.WhenAll(partialInternalFileReps.Select(f => DeleteFileAsync(f.FileId, cancellationToken)));
}
return(fileId);
}
/// <inheritdoc />
public Task <bool> VerifyChecksumAsync(string fileId, string algorithm, byte[] checksum, CancellationToken cancellationToken)
{
bool valid;
var internalFileId = new InternalFileId(fileId);
using (var dataStream = _fileRepFactory.Data(internalFileId).GetStream(FileMode.Open, FileAccess.ReadWrite, FileShare.Read))
{
var chunkPositionFile = _fileRepFactory.ChunkStartPosition(internalFileId);
var chunkStartPosition = chunkPositionFile.ReadFirstLineAsLong(true, 0);
var calculateSha1 = dataStream.CalculateSha1(chunkStartPosition);
valid = checksum.SequenceEqual(calculateSha1);
if (!valid)
{
dataStream.Seek(0, SeekOrigin.Begin);
dataStream.SetLength(chunkStartPosition);
chunkPositionFile.Delete();
_fileRepFactory.ChunkComplete(internalFileId).Delete();
}
}
return(Task.FromResult(valid));
}
/// <inheritdoc />
public async Task SetUploadLengthAsync(string fileId, long uploadLength, CancellationToken _)
{
_fileRepFactory.UploadLength(await InternalFileId.Parse(_fileIdProvider, fileId)).Write(uploadLength.ToString());
}
public InternalFileRep UploadConcat(InternalFileId fileId) => Create(fileId, "uploadconcat");
public InternalFileRep UploadLength(InternalFileId fileId) => Create(fileId, "uploadlength");
public InternalFileRep Metadata(InternalFileId fileId) => Create(fileId, "metadata");
/// <inheritdoc />
public async Task SetExpirationAsync(string fileId, DateTimeOffset expires, CancellationToken _)
{
_fileRepFactory.Expiration(await InternalFileId.Parse(_fileIdProvider, fileId)).Write(expires.ToString("O"));
}
public InternalFileRep Expiration(InternalFileId fileId) => Create(fileId, "expiration");
public InternalFileRep ChunkStartPosition(InternalFileId fileId) => Create(fileId, "chunkstart");
public InternalFileRep ChunkComplete(InternalFileId fileId) => Create(fileId, "chunkcomplete");
/// <inheritdoc />
public async Task <bool> FileExistAsync(string fileId, CancellationToken _)
{
return(_fileRepFactory.Data(await InternalFileId.Parse(_fileIdProvider, fileId)).Exist());
}
/// <inheritdoc />
public async Task <string> GetUploadMetadataAsync(string fileId, CancellationToken _)
{
var firstLine = _fileRepFactory.Metadata(await InternalFileId.Parse(_fileIdProvider, fileId)).ReadFirstLine(true);
return(string.IsNullOrEmpty(firstLine) ? null : firstLine);
}
/// <inheritdoc />
public async Task <long> AppendDataAsync(string fileId, Stream stream, CancellationToken cancellationToken)
{
var internalFileId = await InternalFileId.Parse(_fileIdProvider, fileId);
var httpReadBuffer = _bufferPool.Rent(_maxReadBufferSize);
var fileWriteBuffer = _bufferPool.Rent(Math.Max(_maxWriteBufferSize, _maxReadBufferSize));
try
{
var fileUploadLengthProvidedDuringCreate = await GetUploadLengthAsync(fileId, cancellationToken);
using var diskFileStream = _fileRepFactory.Data(internalFileId).GetStream(FileMode.Append, FileAccess.Write, FileShare.None);
var totalDiskFileLength = diskFileStream.Length;
if (fileUploadLengthProvidedDuringCreate == totalDiskFileLength)
{
return(0);
}
var chunkCompleteFile = InitializeChunk(internalFileId, totalDiskFileLength);
int numberOfbytesReadFromClient;
var bytesWrittenThisRequest = 0L;
var clientDisconnectedDuringRead = false;
var writeBufferNextFreeIndex = 0;
do
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
numberOfbytesReadFromClient = await stream.ReadAsync(httpReadBuffer, 0, _maxReadBufferSize, cancellationToken);
clientDisconnectedDuringRead = cancellationToken.IsCancellationRequested;
totalDiskFileLength += numberOfbytesReadFromClient;
if (totalDiskFileLength > fileUploadLengthProvidedDuringCreate)
{
throw new TusStoreException($"Stream contains more data than the file's upload length. Stream data: {totalDiskFileLength}, upload length: {fileUploadLengthProvidedDuringCreate}.");
}
// Can we fit the read data into the write buffer? If not flush it now.
if (writeBufferNextFreeIndex + numberOfbytesReadFromClient > _maxWriteBufferSize)
{
await FlushFileToDisk(fileWriteBuffer, diskFileStream, writeBufferNextFreeIndex);
writeBufferNextFreeIndex = 0;
}
Array.Copy(
sourceArray: httpReadBuffer,
sourceIndex: 0,
destinationArray: fileWriteBuffer,
destinationIndex: writeBufferNextFreeIndex,
length: numberOfbytesReadFromClient);
writeBufferNextFreeIndex += numberOfbytesReadFromClient;
bytesWrittenThisRequest += numberOfbytesReadFromClient;
} while (numberOfbytesReadFromClient != 0);
// Flush the remaining buffer to disk.
if (writeBufferNextFreeIndex != 0)
{
await FlushFileToDisk(fileWriteBuffer, diskFileStream, writeBufferNextFreeIndex);
}
if (!clientDisconnectedDuringRead)
{
MarkChunkComplete(chunkCompleteFile);
}
return(bytesWrittenThisRequest);
}
finally
{
_bufferPool.Return(httpReadBuffer);
_bufferPool.Return(fileWriteBuffer);
}
}
/// <inheritdoc />
public async Task <long> GetUploadOffsetAsync(string fileId, CancellationToken _)
{
return(_fileRepFactory.Data(await InternalFileId.Parse(_fileIdProvider, fileId)).GetLength());
}
