public async Task <BackblazeApiResponse <UploadPartUrlOutputDTO> > GetUploadPartUrlAsync(
AuthorizationOutputDTO auth,
FileDataOutputDTO uploadData,
CancellationToken cancellationToken = default(CancellationToken)
)
{
var body = SerializeToJsonContent(new UploadPartUrlInputDTO()
{
FileId = uploadData.FileId
});
var request = new HttpRequestMessage
{
Method = HttpMethod.Post,
RequestUri = new Uri($"{auth.ApiUrl}/b2api/v2/b2_get_upload_part_url"),
Content = body
};
request.Headers.TryAddWithoutValidation("Authorization", auth.AuthorizationToken);
var response = await _client.SendAsync(request, cancellationToken);
string responseJson = await response.Content.ReadAsStringAsync();
if (!response.IsSuccessStatusCode)
{
return(new BackblazeApiResponse <UploadPartUrlOutputDTO>()
{
Succeeded = response.IsSuccessStatusCode,
Error = JsonSerializer.Deserialize <B2ErrorResponseOutputDTO>(responseJson)
});
}
return(new BackblazeApiResponse <UploadPartUrlOutputDTO>()
{
Succeeded = response.IsSuccessStatusCode,
Data = JsonSerializer.Deserialize <UploadPartUrlOutputDTO>(responseJson)
});
}
public async Task <BackblazeApiResponse <FileDataOutputDTO>[]> UploadDynamicBatchAsync(
AuthorizationOutputDTO auth,
IEnumerable <UploadFile> files,
IProgress <TransferProgress> progressData = null,
int retryTimeoutCount = 5,
int concurrentUploads = 4,
string tempDir = "/tmp",
CancellationToken cancellationToken = default(CancellationToken)
)
{
FileDataOutputDTO[] result = new FileDataOutputDTO[files.Count()];
Task <BackblazeApiResponse <FileDataOutputDTO> >[] uploadTasks = new Task <BackblazeApiResponse <FileDataOutputDTO> > [files.Count()];
Dictionary <string, FileProgress> fileProgress = new Dictionary <string, FileProgress>();
foreach (var file in files)
{
long fileSize = new FileInfo(file.sourceFilePath).Length;
fileProgress.Add(file.destinationFilename, new FileProgress(filename: file.destinationFilename, totalBytes: fileSize));
}
int fileI = 0;
foreach (var file in files)
{
long fileSize = new FileInfo(file.sourceFilePath).Length;
if (fileSize > auth.RecommendedPartSize)
{
uploadTasks[fileI] = ProcessLargeUploadAsync(
auth: auth,
file: file,
fileProgress: fileProgress,
fileSize: fileSize,
progressData: progressData,
retryTimeoutCount: retryTimeoutCount,
cancellationToken: cancellationToken
);
}
else
{
uploadTasks[fileI] = ProcessSmallUploadAsync(
auth: auth,
file: file,
fileProgress: fileProgress,
progressData: progressData,
retryTimeoutCount: retryTimeoutCount,
cancellationToken: cancellationToken
);
}
fileI++;
}
SemaphoreSlim throttler = new SemaphoreSlim(concurrentUploads, concurrentUploads);
IEnumerable <Task <BackblazeApiResponse <FileDataOutputDTO> > > tasks = uploadTasks.Select(async input =>
{
await throttler.WaitAsync();
try
{
return(await input);
}
finally
{
throttler.Release();
}
});
return(await Task.WhenAll(tasks));
}
public async Task <BackblazeApiResponse <UploadedPartOutputDTO> > UploadLargeFilePartAsync(
AuthorizationOutputDTO auth,
FileDataOutputDTO fileData,
Stream partStream,
int partNumber,
int retryTimeoutCount = 5,
IProgress <FilePartProgress> progressData = null,
CancellationToken cancellationToken = default(CancellationToken)
)
{
if (partStream == null)
{
throw new NullReferenceException("partStream");
}
if (!partStream.CanRead)
{
throw new InvalidDataException();
}
int retryCount = 0;
Retry:
var uploadUrlResponse = await GetUploadPartUrlAsync(auth, fileData, cancellationToken);
if (!uploadUrlResponse.Succeeded)
{
return(new BackblazeApiResponse <UploadedPartOutputDTO>()
{
Succeeded = uploadUrlResponse.Succeeded,
Error = uploadUrlResponse.Error
});
}
UploadedPartOutputDTO uploadPart = null;
var uploadPartResponse = await UploadLargeFilePartAsync(uploadUrlResponse.Data, partStream, fileData.FileName, partNumber, progressData, cancellationToken);
if (uploadPartResponse.Succeeded)
{
uploadPart = uploadPartResponse.Data;
}
else
{
if (ErrorHelper.IsRecoverableError(uploadPartResponse.Error))
{
if (retryCount > retryTimeoutCount)
{
uploadPartResponse.Error.Message = $"Hit retry max on error: {uploadPartResponse.Error.Message}";
return(new BackblazeApiResponse <UploadedPartOutputDTO>()
{
Succeeded = false,
Error = uploadPartResponse.Error
});
}
retryCount++;
goto Retry;
}
else
{
return(new BackblazeApiResponse <UploadedPartOutputDTO>()
{
Succeeded = false,
Error = uploadPartResponse.Error
});
}
}
return(new BackblazeApiResponse <UploadedPartOutputDTO>()
{
Succeeded = uploadUrlResponse.Succeeded,
Data = uploadPart
});
}
/// <summary>
/// Upload a file with attempted recovery from errors
/// </summary>
public async Task <BackblazeApiResponse <FileDataOutputDTO> > UploadSmallFileAsync(
AuthorizationOutputDTO auth,
string bucketId,
string sourcePath,
string destinationFilename,
int retryTimeoutCount = 5,
IProgress <FileProgress> progressData = null,
CancellationToken cancellationToken = default(CancellationToken)
)
{
int retryCount = 0;
Retry:
var uploadUrlResponse = await GetUploadUrlAsync(auth, bucketId, cancellationToken);
if (!uploadUrlResponse.Succeeded)
{
return(new BackblazeApiResponse <FileDataOutputDTO>()
{
Succeeded = uploadUrlResponse.Succeeded,
Error = uploadUrlResponse.Error
});
}
FileDataOutputDTO uploadData = null;
var uploadDataResponse = await UploadSmallFileAsync(uploadUrlResponse.Data, sourcePath, destinationFilename, progressData, cancellationToken);
if (uploadDataResponse.Succeeded)
{
uploadData = uploadDataResponse.Data;
}
else
{
if (ErrorHelper.IsRecoverableError(uploadDataResponse.Error))
{
if (retryCount > retryTimeoutCount)
{
uploadDataResponse.Error.Message = $"Hit retry max on error: {uploadDataResponse.Error.Message}";
return(new BackblazeApiResponse <FileDataOutputDTO>()
{
Succeeded = false,
Error = uploadDataResponse.Error
});
}
retryCount++;
goto Retry;
}
else
{
return(new BackblazeApiResponse <FileDataOutputDTO>()
{
Succeeded = false,
Error = uploadDataResponse.Error
});
}
}
return(new BackblazeApiResponse <FileDataOutputDTO>()
{
Succeeded = uploadDataResponse.Succeeded,
Data = uploadData
});
}
