private async Task <BackblazeApiResponse <FileDataOutputDTO> > ProcessSmallUploadAsync( AuthorizationOutputDTO auth, UploadFile file, IDictionary <string, FileProgress> fileProgress, IProgress <TransferProgress> progressData = null, int retryTimeoutCount = 5, CancellationToken cancellationToken = default(CancellationToken) ) { var result = await B2Client.UploadSmallFileAsync( auth : auth, bucketId : file.bucketId, sourcePath : file.sourceFilePath, destinationFilename : file.destinationFilename, retryTimeoutCount : retryTimeoutCount, cancellationToken : cancellationToken, progressData : new Progress <FileProgress>((FileProgress progress) => { fileProgress[progress.filename] = progress; progressData?.Report(new TransferProgress( fileProgress: new Dictionary <string, FileProgress>(fileProgress) )); }) ); return(result); }
public async Task <BackblazeApiResponse <ListFileVersionsOutputDTO> > ListFileVersionsAsync( AuthorizationOutputDTO auth, string bucketId, string startFileName = null, string startFileId = null, int?maxFileCount = null, string prefix = null, string delimiter = null, CancellationToken cancellationToken = default(CancellationToken) ) { var body = SerializeToJsonContent(new ListFileVersionsInputDTO() { BucketId = bucketId, StartFileName = startFileName, StartFileId = startFileId, MaxFileCount = maxFileCount, Prefix = prefix, Delimiter = delimiter, }); var request = new HttpRequestMessage { Method = HttpMethod.Post, RequestUri = new Uri($"{auth.ApiUrl}/b2api/v2/b2_list_file_versions"), Content = body }; request.Headers.TryAddWithoutValidation("Authorization", auth.AuthorizationToken); var response = await _client.SendAsync(request, cancellationToken); string responseJson = await response.Content.ReadAsStringAsync(); if (!response.IsSuccessStatusCode) { return(new BackblazeApiResponse <ListFileVersionsOutputDTO>() { Succeeded = response.IsSuccessStatusCode, Error = JsonSerializer.Deserialize <B2ErrorResponseOutputDTO>(responseJson) }); } return(new BackblazeApiResponse <ListFileVersionsOutputDTO>() { Succeeded = response.IsSuccessStatusCode, Data = JsonSerializer.Deserialize <ListFileVersionsOutputDTO>(responseJson) }); }
public async Task <BackblazeApiResponse <FileDataOutputDTO> > UploadDynamicAsync( AuthorizationOutputDTO auth, UploadFile file, IProgress <TransferProgress> progressData = null, int retryTimeoutCount = 5, int concurrentUploads = 4, string tempDir = "/tmp", CancellationToken cancellationToken = default(CancellationToken) ) { ConcurrentDictionary <string, FileProgress> fileProgress = new ConcurrentDictionary <string, FileProgress>(); long fileSize = new FileInfo(file.sourceFilePath).Length; fileProgress.TryAdd(file.destinationFilename, new FileProgress(filename: file.destinationFilename, totalBytes: fileSize)); Task <BackblazeApiResponse <FileDataOutputDTO> > uploadTask; if (fileSize > auth.RecommendedPartSize) { uploadTask = ProcessLargeUploadAsync( auth: auth, file: file, fileProgress: fileProgress, fileSize: fileSize, progressData: progressData, retryTimeoutCount: retryTimeoutCount, cancellationToken: cancellationToken ); } else { uploadTask = ProcessSmallUploadAsync( auth: auth, file: file, fileProgress: fileProgress, progressData: progressData, retryTimeoutCount: retryTimeoutCount, cancellationToken: cancellationToken ); } return(await uploadTask); }
public async Task <BackblazeApiResponse <FileVersionDeletedOutputDTO> > DeleteFileVersionAsync( AuthorizationOutputDTO auth, string fileName, string fileId, bool?bypassGovernance = null, CancellationToken cancellationToken = default(CancellationToken) ) { var body = SerializeToJsonContent(new DeleteFileVersionInputDTO() { FileName = fileName, FileId = fileId, BypassGovernance = bypassGovernance, }); var request = new HttpRequestMessage { Method = HttpMethod.Post, RequestUri = new Uri($"{auth.ApiUrl}/b2api/v2/b2_delete_file_version"), Content = body }; request.Headers.TryAddWithoutValidation("Authorization", auth.AuthorizationToken); var response = await _client.SendAsync(request, cancellationToken); string responseJson = await response.Content.ReadAsStringAsync(); if (!response.IsSuccessStatusCode) { return(new BackblazeApiResponse <FileVersionDeletedOutputDTO>() { Succeeded = response.IsSuccessStatusCode, Error = JsonSerializer.Deserialize <B2ErrorResponseOutputDTO>(responseJson) }); } return(new BackblazeApiResponse <FileVersionDeletedOutputDTO>() { Succeeded = response.IsSuccessStatusCode, Data = JsonSerializer.Deserialize <FileVersionDeletedOutputDTO>(responseJson) }); }
public async Task <BackblazeApiResponse <FileDataOutputDTO> > StartLargeFileAsync( AuthorizationOutputDTO auth, string bucketId, string destinationFilename, CancellationToken cancellationToken = default(CancellationToken) ) { var body = SerializeToJsonContent(new StartLargeFileUploadInputDTO() { BucketId = bucketId, FileName = destinationFilename, ContentType = "b2/x-auto", }); var request = new HttpRequestMessage { Method = HttpMethod.Post, RequestUri = new Uri($"{auth.ApiUrl}/b2api/v2/b2_start_large_file"), Content = body }; request.Headers.TryAddWithoutValidation("Authorization", auth.AuthorizationToken); var response = await _client.SendAsync(request, cancellationToken); string responseJson = await response.Content.ReadAsStringAsync(); if (!response.IsSuccessStatusCode) { return(new BackblazeApiResponse <FileDataOutputDTO>() { Succeeded = response.IsSuccessStatusCode, Error = JsonSerializer.Deserialize <B2ErrorResponseOutputDTO>(responseJson) }); } return(new BackblazeApiResponse <FileDataOutputDTO>() { Succeeded = response.IsSuccessStatusCode, Data = JsonSerializer.Deserialize <FileDataOutputDTO>(responseJson) }); }
public async Task <BackblazeApiResponse <FileDataOutputDTO> > FinishLargeFileAsync( AuthorizationOutputDTO auth, string fileId, IEnumerable <string> partHashes, CancellationToken cancellationToken = default(CancellationToken) ) { var body = SerializeToJsonContent(new FinishLargeFileUploadInputDTO() { FileId = fileId, PartSha1Array = (string[])partHashes }); var request = new HttpRequestMessage { Method = HttpMethod.Post, RequestUri = new Uri($"{auth.ApiUrl}/b2api/v2/b2_finish_large_file"), Content = body }; request.Headers.TryAddWithoutValidation("Authorization", auth.AuthorizationToken); var response = await _client.SendAsync(request, cancellationToken); string responseJson = await response.Content.ReadAsStringAsync(); if (!response.IsSuccessStatusCode) { return(new BackblazeApiResponse <FileDataOutputDTO>() { Succeeded = response.IsSuccessStatusCode, Error = JsonSerializer.Deserialize <B2ErrorResponseOutputDTO>(responseJson) }); } return(new BackblazeApiResponse <FileDataOutputDTO>() { Succeeded = response.IsSuccessStatusCode, Data = JsonSerializer.Deserialize <FileDataOutputDTO>(responseJson) }); }
public async Task <BackblazeApiResponse <UploadUrlOutputDTO> > GetUploadUrlAsync( AuthorizationOutputDTO auth, string bucketId, CancellationToken cancellationToken = default(CancellationToken) ) { var body = SerializeToJsonContent(new UploadUrlInputDTO() { BucketId = bucketId }); var request = new HttpRequestMessage { Method = HttpMethod.Post, RequestUri = new Uri($"{auth.ApiUrl}/b2api/v2/b2_get_upload_url"), Content = body }; request.Headers.TryAddWithoutValidation("Authorization", auth.AuthorizationToken); var response = await _client.SendAsync(request, cancellationToken); string responseJson = await response.Content.ReadAsStringAsync(); if (!response.IsSuccessStatusCode) { return(new BackblazeApiResponse <UploadUrlOutputDTO>() { Succeeded = response.IsSuccessStatusCode, Error = JsonSerializer.Deserialize <B2ErrorResponseOutputDTO>(responseJson) }); } return(new BackblazeApiResponse <UploadUrlOutputDTO>() { Succeeded = response.IsSuccessStatusCode, Data = JsonSerializer.Deserialize <UploadUrlOutputDTO>(responseJson) }); }
public async Task <BackblazeApiResponse <FileDataOutputDTO>[]> UploadDynamicBatchAsync( AuthorizationOutputDTO auth, IEnumerable <UploadFile> files, IProgress <TransferProgress> progressData = null, int retryTimeoutCount = 5, int concurrentUploads = 4, string tempDir = "/tmp", CancellationToken cancellationToken = default(CancellationToken) ) { FileDataOutputDTO[] result = new FileDataOutputDTO[files.Count()]; Task <BackblazeApiResponse <FileDataOutputDTO> >[] uploadTasks = new Task <BackblazeApiResponse <FileDataOutputDTO> > [files.Count()]; Dictionary <string, FileProgress> fileProgress = new Dictionary <string, FileProgress>(); foreach (var file in files) { long fileSize = new FileInfo(file.sourceFilePath).Length; fileProgress.Add(file.destinationFilename, new FileProgress(filename: file.destinationFilename, totalBytes: fileSize)); } int fileI = 0; foreach (var file in files) { long fileSize = new FileInfo(file.sourceFilePath).Length; if (fileSize > auth.RecommendedPartSize) { uploadTasks[fileI] = ProcessLargeUploadAsync( auth: auth, file: file, fileProgress: fileProgress, fileSize: fileSize, progressData: progressData, retryTimeoutCount: retryTimeoutCount, cancellationToken: cancellationToken ); } else { uploadTasks[fileI] = ProcessSmallUploadAsync( auth: auth, file: file, fileProgress: fileProgress, progressData: progressData, retryTimeoutCount: retryTimeoutCount, cancellationToken: cancellationToken ); } fileI++; } SemaphoreSlim throttler = new SemaphoreSlim(concurrentUploads, concurrentUploads); IEnumerable <Task <BackblazeApiResponse <FileDataOutputDTO> > > tasks = uploadTasks.Select(async input => { await throttler.WaitAsync(); try { return(await input); } finally { throttler.Release(); } }); return(await Task.WhenAll(tasks)); }
private async Task <BackblazeApiResponse <FileDataOutputDTO> > ProcessLargeUploadAsync( AuthorizationOutputDTO auth, UploadFile file, IDictionary <string, FileProgress> fileProgress, long fileSize, IProgress <TransferProgress> progressData = null, int retryTimeoutCount = 5, string tempDir = "/tmp", CancellationToken cancellationToken = default(CancellationToken) ) { int partCount = (int)Math.Ceiling((double)fileSize / (double)auth.RecommendedPartSize); var initDataResponse = await B2Client.StartLargeFileAsync(auth, file.bucketId, file.destinationFilename, cancellationToken); if (!initDataResponse.Succeeded) { return(new BackblazeApiResponse <FileDataOutputDTO>() { Succeeded = false, Error = initDataResponse.Error }); } string[] partHashes = new string[partCount]; List <UploadedPartOutputDTO> completeParts = new List <UploadedPartOutputDTO>(); for (int i = 0; i < partCount; i++) { int partNumber = i + 1; long partSize = auth.RecommendedPartSize; if (partNumber == partCount) { partSize = fileSize - ((partCount - 1) * auth.RecommendedPartSize); } long offset = i * auth.RecommendedPartSize; string tmpFilePath = Path.Combine(tempDir, Guid.NewGuid().ToString()); FileSplitter.Extract(file.sourceFilePath, tmpFilePath, offset, partSize); using (var stream = System.IO.File.OpenRead(tmpFilePath)) { var uploadPartResponse = await B2Client.UploadLargeFilePartAsync( auth : auth, fileData : initDataResponse.Data, partStream : stream, partNumber : partNumber, retryTimeoutCount : retryTimeoutCount, cancellationToken : cancellationToken, progressData : new Progress <FilePartProgress>((FilePartProgress progress) => { var oldProgress = fileProgress[progress.filename]; long completeBytes = 0; foreach (var completePart in completeParts) { completeBytes += completePart.ContentLength; } var newProgress = new FileProgress( filename: progress.filename, totalBytes: oldProgress.totalBytes, bytesTransferred: completeBytes + progress.bytesTransferred ); fileProgress[progress.filename] = newProgress; progressData?.Report(new TransferProgress( fileProgress: new Dictionary <string, FileProgress>(fileProgress) )); }) ); stream.Close(); if (!uploadPartResponse.Succeeded) { throw new B2Exception(uploadPartResponse.Error); } else { completeParts.Add(uploadPartResponse.Data); partHashes[i] = uploadPartResponse.Data.ContentSha1; } } File.Delete(tmpFilePath); } return(await B2Client.FinishLargeFileAsync(auth, initDataResponse.Data.FileId, partHashes, cancellationToken)); }
public async Task <BackblazeApiResponse <UploadedPartOutputDTO> > UploadLargeFilePartAsync( AuthorizationOutputDTO auth, FileDataOutputDTO fileData, Stream partStream, int partNumber, int retryTimeoutCount = 5, IProgress <FilePartProgress> progressData = null, CancellationToken cancellationToken = default(CancellationToken) ) { if (partStream == null) { throw new NullReferenceException("partStream"); } if (!partStream.CanRead) { throw new InvalidDataException(); } int retryCount = 0; Retry: var uploadUrlResponse = await GetUploadPartUrlAsync(auth, fileData, cancellationToken); if (!uploadUrlResponse.Succeeded) { return(new BackblazeApiResponse <UploadedPartOutputDTO>() { Succeeded = uploadUrlResponse.Succeeded, Error = uploadUrlResponse.Error }); } UploadedPartOutputDTO uploadPart = null; var uploadPartResponse = await UploadLargeFilePartAsync(uploadUrlResponse.Data, partStream, fileData.FileName, partNumber, progressData, cancellationToken); if (uploadPartResponse.Succeeded) { uploadPart = uploadPartResponse.Data; } else { if (ErrorHelper.IsRecoverableError(uploadPartResponse.Error)) { if (retryCount > retryTimeoutCount) { uploadPartResponse.Error.Message = $"Hit retry max on error: {uploadPartResponse.Error.Message}"; return(new BackblazeApiResponse <UploadedPartOutputDTO>() { Succeeded = false, Error = uploadPartResponse.Error }); } retryCount++; goto Retry; } else { return(new BackblazeApiResponse <UploadedPartOutputDTO>() { Succeeded = false, Error = uploadPartResponse.Error }); } } return(new BackblazeApiResponse <UploadedPartOutputDTO>() { Succeeded = uploadUrlResponse.Succeeded, Data = uploadPart }); }
/// <summary> /// Upload a file with attempted recovery from errors /// </summary> public async Task <BackblazeApiResponse <FileDataOutputDTO> > UploadSmallFileAsync( AuthorizationOutputDTO auth, string bucketId, string sourcePath, string destinationFilename, int retryTimeoutCount = 5, IProgress <FileProgress> progressData = null, CancellationToken cancellationToken = default(CancellationToken) ) { int retryCount = 0; Retry: var uploadUrlResponse = await GetUploadUrlAsync(auth, bucketId, cancellationToken); if (!uploadUrlResponse.Succeeded) { return(new BackblazeApiResponse <FileDataOutputDTO>() { Succeeded = uploadUrlResponse.Succeeded, Error = uploadUrlResponse.Error }); } FileDataOutputDTO uploadData = null; var uploadDataResponse = await UploadSmallFileAsync(uploadUrlResponse.Data, sourcePath, destinationFilename, progressData, cancellationToken); if (uploadDataResponse.Succeeded) { uploadData = uploadDataResponse.Data; } else { if (ErrorHelper.IsRecoverableError(uploadDataResponse.Error)) { if (retryCount > retryTimeoutCount) { uploadDataResponse.Error.Message = $"Hit retry max on error: {uploadDataResponse.Error.Message}"; return(new BackblazeApiResponse <FileDataOutputDTO>() { Succeeded = false, Error = uploadDataResponse.Error }); } retryCount++; goto Retry; } else { return(new BackblazeApiResponse <FileDataOutputDTO>() { Succeeded = false, Error = uploadDataResponse.Error }); } } return(new BackblazeApiResponse <FileDataOutputDTO>() { Succeeded = uploadDataResponse.Succeeded, Data = uploadData }); }