/// <inheritdoc /> public async Task <string> CreatePartialFileAsync(long uploadLength, string metadata, CancellationToken cancellationToken) { var fileId = await CreateFileAsync(uploadLength, metadata, cancellationToken); _fileRepFactory.UploadConcat(await InternalFileId.Parse(_fileIdProvider, fileId)).Write(new FileConcatPartial().GetHeader()); return(fileId); }
/// <inheritdoc /> public async Task <bool> VerifyChecksumAsync(string fileId, string algorithm, byte[] checksum, CancellationToken _) { var valid = false; var internalFileId = await InternalFileId.Parse(_fileIdProvider, fileId); using (var dataStream = _fileRepFactory.Data(internalFileId).GetStream(FileMode.Open, FileAccess.ReadWrite, FileShare.Read)) { var chunkPositionFile = _fileRepFactory.ChunkStartPosition(internalFileId); var chunkStartPosition = chunkPositionFile.ReadFirstLineAsLong(true, 0); var chunkCompleteFile = _fileRepFactory.ChunkComplete(internalFileId); // Only verify the checksum if the entire lastest chunk has been written. // If not, just discard the last chunk as it won't match the checksum anyway. if (chunkCompleteFile.Exist()) { var calculateSha1 = dataStream.CalculateSha1(chunkStartPosition); valid = checksum.SequenceEqual(calculateSha1); } if (!valid) { dataStream.Seek(0, SeekOrigin.Begin); dataStream.SetLength(chunkStartPosition); } } return(valid); }
/// <inheritdoc /> public async Task <long?> GetUploadLengthAsync(string fileId, CancellationToken _) { var firstLine = _fileRepFactory.UploadLength(await InternalFileId.Parse(_fileIdProvider, fileId)).ReadFirstLine(true); return(firstLine == null ? (long?)null : long.Parse(firstLine)); }
/// <inheritdoc /> public async Task <FileConcat> GetUploadConcatAsync(string fileId, CancellationToken _) { var firstLine = _fileRepFactory.UploadConcat(await InternalFileId.Parse(_fileIdProvider, fileId)).ReadFirstLine(true); return(string.IsNullOrWhiteSpace(firstLine) ? null : new UploadConcat(firstLine).Type); }
/// <inheritdoc /> public async Task <DateTimeOffset?> GetExpirationAsync(string fileId, CancellationToken _) { var expiration = _fileRepFactory.Expiration(await InternalFileId.Parse(_fileIdProvider, fileId)).ReadFirstLine(true); return(expiration == null ? (DateTimeOffset?)null : DateTimeOffset.ParseExact(expiration, "O", null)); }
private InternalFileRep InitializeChunk(InternalFileId internalFileId, long totalDiskFileLength) { var chunkComplete = _fileRepFactory.ChunkComplete(internalFileId); chunkComplete.Delete(); _fileRepFactory.ChunkStartPosition(internalFileId).Write(totalDiskFileLength.ToString()); return(chunkComplete); }
/// <inheritdoc /> public Task <ITusFile> GetFileAsync(string fileId, CancellationToken cancellationToken) { var internalFileId = new InternalFileId(fileId); var data = _fileRepFactory.Data(internalFileId); return(Task.FromResult <ITusFile>(data.Exist() ? new TusDiskFile(data, _fileRepFactory.Metadata(internalFileId)) : null)); }
/// <inheritdoc /> public async Task <long> AppendDataAsync(string fileId, Stream stream, CancellationToken cancellationToken) { var internalFileId = new InternalFileId(fileId); long bytesWritten = 0; var uploadLength = await GetUploadLengthAsync(fileId, cancellationToken); using (var file = _fileRepFactory.Data(internalFileId).GetStream(FileMode.Append, FileAccess.Write, FileShare.None)) { var fileLength = file.Length; if (uploadLength == fileLength) { return(0); } var chunkStart = _fileRepFactory.ChunkStartPosition(internalFileId); var chunkComplete = _fileRepFactory.ChunkComplete(internalFileId); if (chunkComplete.Exist()) { chunkStart.Delete(); chunkComplete.Delete(); } if (!chunkStart.Exist()) { chunkStart.Write(fileLength.ToString()); } int bytesRead; do { if (cancellationToken.IsCancellationRequested) { break; } var buffer = new byte[ByteChunkSize]; bytesRead = await stream.ReadAsync(buffer, 0, ByteChunkSize, cancellationToken); fileLength += bytesRead; if (fileLength > uploadLength) { throw new TusStoreException( $"Stream contains more data than the file's upload length. Stream data: {fileLength}, upload length: {uploadLength}."); } file.Write(buffer, 0, bytesRead); bytesWritten += bytesRead; } while (bytesRead != 0); // Chunk is complete. Mark it as complete. chunkComplete.Write("1"); return(bytesWritten); } }
/// <inheritdoc /> public async Task <ITusFile> GetFileAsync(string fileId, CancellationToken _) { var internalFileId = await InternalFileId.Parse(_fileIdProvider, fileId); var data = _fileRepFactory.Data(internalFileId); return(data.Exist() ? new TusDiskFile(data, _fileRepFactory.Metadata(internalFileId)) : null); }
private InternalFileRep Create(InternalFileId fileId, string extension) { string fileName = fileId; if (!string.IsNullOrEmpty(extension)) { fileName += "." + extension; } return(new InternalFileRep(fileId, System.IO.Path.Combine(_directoryPath, fileName))); }
/// <inheritdoc /> public async Task <string> CreateFileAsync(long uploadLength, string metadata, CancellationToken cancellationToken) { var fileId = await InternalFileId.CreateNew(_fileIdProvider, metadata); new FileStream(_fileRepFactory.Data(fileId).Path, FileMode.CreateNew).Dispose(); if (uploadLength != -1) { await SetUploadLengthAsync(fileId, uploadLength, cancellationToken); } _fileRepFactory.Metadata(fileId).Write(metadata); return(fileId); }
/// <inheritdoc /> public async Task <string> CreateFileAsync(long uploadLength, string metadata, CancellationToken cancellationToken) { var fileId = new InternalFileId(); File.Create(_fileRepFactory.Data(fileId).Path).Dispose(); if (uploadLength != -1) { await SetUploadLengthAsync(fileId.FileId, uploadLength, cancellationToken); } _fileRepFactory.Metadata(fileId).Write(metadata); return(fileId.FileId); }
/// <inheritdoc /> public async Task DeleteFileAsync(string fileId, CancellationToken _) { var internalFileId = await InternalFileId.Parse(_fileIdProvider, fileId); await Task.Run(() => { _fileRepFactory.Data(internalFileId).Delete(); _fileRepFactory.UploadLength(internalFileId).Delete(); _fileRepFactory.Metadata(internalFileId).Delete(); _fileRepFactory.UploadConcat(internalFileId).Delete(); _fileRepFactory.ChunkStartPosition(internalFileId).Delete(); _fileRepFactory.ChunkComplete(internalFileId).Delete(); _fileRepFactory.Expiration(internalFileId).Delete(); }); }
/// <inheritdoc /> public Task DeleteFileAsync(string fileId, CancellationToken cancellationToken) { var internalFileId = new InternalFileId(fileId); return(Task.Run(() => { _fileRepFactory.Data(internalFileId).Delete(); _fileRepFactory.UploadLength(internalFileId).Delete(); _fileRepFactory.Metadata(internalFileId).Delete(); _fileRepFactory.UploadConcat(internalFileId).Delete(); _fileRepFactory.Expiration(internalFileId).Delete(); _fileRepFactory.ChunkStartPosition(internalFileId).Delete(); _fileRepFactory.ChunkComplete(internalFileId).Delete(); }, cancellationToken)); }
/// <inheritdoc /> public async Task <IEnumerable <string> > GetExpiredFilesAsync(CancellationToken _) { List <string> expiredFiles = new List <string>(); foreach (var file in Directory.EnumerateFiles(_directoryPath, "*.expiration")) { var f = await InternalFileId.Parse(_fileIdProvider, Path.GetFileNameWithoutExtension(file)); if (FileHasExpired(f) && FileIsIncomplete(f)) { expiredFiles.Add(f); } } return(expiredFiles); bool FileHasExpired(InternalFileId fileId) { var firstLine = _fileRepFactory.Expiration(fileId).ReadFirstLine(); return(!string.IsNullOrWhiteSpace(firstLine) && DateTimeOffset.ParseExact(firstLine, "O", null).HasPassed()); } bool FileIsIncomplete(InternalFileId fileId) { var uploadLength = _fileRepFactory.UploadLength(fileId).ReadFirstLineAsLong(fileIsOptional: true, defaultValue: long.MinValue); if (uploadLength == long.MinValue) { return(true); } var dataFile = _fileRepFactory.Data(fileId); if (!dataFile.Exist()) { return(true); } return(uploadLength != dataFile.GetLength()); } }
/// <inheritdoc /> public async Task <string> CreateFinalFileAsync(string[] partialFiles, string metadata, CancellationToken cancellationToken) { var partialInternalFileReps = partialFiles.Select(f => { var partialData = _fileRepFactory.Data(new InternalFileId(f)); if (!partialData.Exist()) { throw new TusStoreException($"File {f} does not exist"); } return(partialData); }).ToArray(); var length = partialInternalFileReps.Sum(f => f.GetLength()); var fileId = await CreateFileAsync(length, metadata, cancellationToken); var internalFileId = new InternalFileId(fileId); _fileRepFactory.UploadConcat(internalFileId).Write(new FileConcatFinal(partialFiles).GetHeader()); using (var finalFile = _fileRepFactory.Data(internalFileId).GetStream(FileMode.Open, FileAccess.Write, FileShare.None)) { foreach (var partialFile in partialInternalFileReps) { using (var partialStream = partialFile.GetStream(FileMode.Open, FileAccess.Read, FileShare.Read)) { partialStream.CopyTo(finalFile); } } } // ReSharper disable once InvertIf if (_deletePartialFilesOnConcat) { await Task.WhenAll(partialInternalFileReps.Select(f => DeleteFileAsync(f.FileId, cancellationToken))); } return(fileId); }
/// <inheritdoc /> public async Task <string> CreateFinalFileAsync(string[] partialFiles, string metadata, CancellationToken cancellationToken) { InternalFileRep[] partialInternalFileReps = new InternalFileRep[partialFiles.Length]; for (int i = 0; i < partialFiles.Length; i++) { partialInternalFileReps[i] = _fileRepFactory.Data(await InternalFileId.Parse(_fileIdProvider, partialFiles[i])); if (!partialInternalFileReps[i].Exist()) { throw new TusStoreException($"File {partialFiles[i]} does not exist"); } } var length = partialInternalFileReps.Sum(f => f.GetLength()); var fileId = await CreateFileAsync(length, metadata, cancellationToken); var internalFileId = await InternalFileId.Parse(_fileIdProvider, fileId); _fileRepFactory.UploadConcat(internalFileId).Write(new FileConcatFinal(partialFiles).GetHeader()); using (var finalFile = _fileRepFactory.Data(internalFileId).GetStream(FileMode.Open, FileAccess.Write, FileShare.None)) { foreach (var partialFile in partialInternalFileReps) { using var partialStream = partialFile.GetStream(FileMode.Open, FileAccess.Read, FileShare.Read); await partialStream.CopyToAsync(finalFile); } } if (_deletePartialFilesOnConcat) { await Task.WhenAll(partialInternalFileReps.Select(f => DeleteFileAsync(f.FileId, cancellationToken))); } return(fileId); }
/// <inheritdoc /> public Task <bool> VerifyChecksumAsync(string fileId, string algorithm, byte[] checksum, CancellationToken cancellationToken) { bool valid; var internalFileId = new InternalFileId(fileId); using (var dataStream = _fileRepFactory.Data(internalFileId).GetStream(FileMode.Open, FileAccess.ReadWrite, FileShare.Read)) { var chunkPositionFile = _fileRepFactory.ChunkStartPosition(internalFileId); var chunkStartPosition = chunkPositionFile.ReadFirstLineAsLong(true, 0); var calculateSha1 = dataStream.CalculateSha1(chunkStartPosition); valid = checksum.SequenceEqual(calculateSha1); if (!valid) { dataStream.Seek(0, SeekOrigin.Begin); dataStream.SetLength(chunkStartPosition); chunkPositionFile.Delete(); _fileRepFactory.ChunkComplete(internalFileId).Delete(); } } return(Task.FromResult(valid)); }
/// <inheritdoc /> public async Task SetUploadLengthAsync(string fileId, long uploadLength, CancellationToken _) { _fileRepFactory.UploadLength(await InternalFileId.Parse(_fileIdProvider, fileId)).Write(uploadLength.ToString()); }
public InternalFileRep UploadConcat(InternalFileId fileId) => Create(fileId, "uploadconcat");
public InternalFileRep UploadLength(InternalFileId fileId) => Create(fileId, "uploadlength");
public InternalFileRep Metadata(InternalFileId fileId) => Create(fileId, "metadata");
/// <inheritdoc /> public async Task SetExpirationAsync(string fileId, DateTimeOffset expires, CancellationToken _) { _fileRepFactory.Expiration(await InternalFileId.Parse(_fileIdProvider, fileId)).Write(expires.ToString("O")); }
public InternalFileRep Expiration(InternalFileId fileId) => Create(fileId, "expiration");
public InternalFileRep ChunkStartPosition(InternalFileId fileId) => Create(fileId, "chunkstart");
public InternalFileRep ChunkComplete(InternalFileId fileId) => Create(fileId, "chunkcomplete");
/// <inheritdoc /> public async Task <bool> FileExistAsync(string fileId, CancellationToken _) { return(_fileRepFactory.Data(await InternalFileId.Parse(_fileIdProvider, fileId)).Exist()); }
/// <inheritdoc /> public async Task <string> GetUploadMetadataAsync(string fileId, CancellationToken _) { var firstLine = _fileRepFactory.Metadata(await InternalFileId.Parse(_fileIdProvider, fileId)).ReadFirstLine(true); return(string.IsNullOrEmpty(firstLine) ? null : firstLine); }
/// <inheritdoc /> public async Task <long> AppendDataAsync(string fileId, Stream stream, CancellationToken cancellationToken) { var internalFileId = await InternalFileId.Parse(_fileIdProvider, fileId); var httpReadBuffer = _bufferPool.Rent(_maxReadBufferSize); var fileWriteBuffer = _bufferPool.Rent(Math.Max(_maxWriteBufferSize, _maxReadBufferSize)); try { var fileUploadLengthProvidedDuringCreate = await GetUploadLengthAsync(fileId, cancellationToken); using var diskFileStream = _fileRepFactory.Data(internalFileId).GetStream(FileMode.Append, FileAccess.Write, FileShare.None); var totalDiskFileLength = diskFileStream.Length; if (fileUploadLengthProvidedDuringCreate == totalDiskFileLength) { return(0); } var chunkCompleteFile = InitializeChunk(internalFileId, totalDiskFileLength); int numberOfbytesReadFromClient; var bytesWrittenThisRequest = 0L; var clientDisconnectedDuringRead = false; var writeBufferNextFreeIndex = 0; do { if (cancellationToken.IsCancellationRequested) { break; } numberOfbytesReadFromClient = await stream.ReadAsync(httpReadBuffer, 0, _maxReadBufferSize, cancellationToken); clientDisconnectedDuringRead = cancellationToken.IsCancellationRequested; totalDiskFileLength += numberOfbytesReadFromClient; if (totalDiskFileLength > fileUploadLengthProvidedDuringCreate) { throw new TusStoreException($"Stream contains more data than the file's upload length. Stream data: {totalDiskFileLength}, upload length: {fileUploadLengthProvidedDuringCreate}."); } // Can we fit the read data into the write buffer? If not flush it now. if (writeBufferNextFreeIndex + numberOfbytesReadFromClient > _maxWriteBufferSize) { await FlushFileToDisk(fileWriteBuffer, diskFileStream, writeBufferNextFreeIndex); writeBufferNextFreeIndex = 0; } Array.Copy( sourceArray: httpReadBuffer, sourceIndex: 0, destinationArray: fileWriteBuffer, destinationIndex: writeBufferNextFreeIndex, length: numberOfbytesReadFromClient); writeBufferNextFreeIndex += numberOfbytesReadFromClient; bytesWrittenThisRequest += numberOfbytesReadFromClient; } while (numberOfbytesReadFromClient != 0); // Flush the remaining buffer to disk. if (writeBufferNextFreeIndex != 0) { await FlushFileToDisk(fileWriteBuffer, diskFileStream, writeBufferNextFreeIndex); } if (!clientDisconnectedDuringRead) { MarkChunkComplete(chunkCompleteFile); } return(bytesWrittenThisRequest); } finally { _bufferPool.Return(httpReadBuffer); _bufferPool.Return(fileWriteBuffer); } }
/// <inheritdoc /> public async Task <long> GetUploadOffsetAsync(string fileId, CancellationToken _) { return(_fileRepFactory.Data(await InternalFileId.Parse(_fileIdProvider, fileId)).GetLength()); }